xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into feature/table

This commit is contained in:
Shengliang Guan 2020-11-06 15:08:24 +08:00
parent 7fa7be7a8a f1e7073ac0
commit 3cf86c92ce
86 changed files with 6401 additions and 3030 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Jenkinsfile vendored
View File

@ -90,7 +90,6 @@ pipeline {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh ''' sh '''
cd ${WKC}/tests/pytest cd ${WKC}/tests/pytest
./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4
./handle_crash_gen_val_log.sh ./handle_crash_gen_val_log.sh
''' '''
} }

3
src/client/inc/tsclient.h
View File

@ -330,6 +330,7 @@ typedef struct STscObj {
char writeAuth : 1; char writeAuth : 1;
char superAuth : 1; char superAuth : 1;
uint32_t connId; uint32_t connId;
uint64_t rid; // ref ID returned by taosAddRef
struct SSqlObj * pHb; struct SSqlObj * pHb;
struct SSqlObj * sqlList; struct SSqlObj * sqlList;
struct SSqlStream *streamList; struct SSqlStream *streamList;
@ -348,7 +349,7 @@ typedef struct SSqlObj {
void *signature; void *signature;
pthread_t owner; // owner of sql object, by which it is executed pthread_t owner; // owner of sql object, by which it is executed
STscObj *pTscObj; STscObj *pTscObj;
void *pRpcCtx; int64_t rpcRid;
void (*fp)(); void (*fp)();
void (*fetchFp)(); void (*fetchFp)();
void *param; void *param;

16
src/client/src/tscAsync.c
View File

@ -428,6 +428,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
} else { } else {
assert(code == TSDB_CODE_SUCCESS); assert(code == TSDB_CODE_SUCCESS);
} }
// param already freed by other routine and pSql in tscCache when ctrl + c // param already freed by other routine and pSql in tscCache when ctrl + c
if (atomic_load_ptr(&pSql->param) == NULL) { if (atomic_load_ptr(&pSql->param) == NULL) {
return; return;
@ -441,6 +442,20 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex && assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL); pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL);
// tscProcessSql can add error into async res
tscProcessSql(pSql);
return;
} else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) {
tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return;
} else {
assert(code == TSDB_CODE_SUCCESS);
}
assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0));
// tscProcessSql can add error into async res // tscProcessSql can add error into async res
tscProcessSql(pSql); tscProcessSql(pSql);
return; return;
@ -465,7 +480,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
tscResetSqlCmdObj(pCmd, false); tscResetSqlCmdObj(pCmd, false);
code = tsParseSql(pSql, true); code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
return; return;
} else if (code != TSDB_CODE_SUCCESS) { } else if (code != TSDB_CODE_SUCCESS) {

3
src/client/src/tscLocalMerge.c
View File

@ -97,6 +97,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
pCtx->param[2].i64Key = pQueryInfo->order.order; pCtx->param[2].i64Key = pQueryInfo->order.order;
pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
pCtx->param[1].i64Key = pQueryInfo->order.orderColId; pCtx->param[1].i64Key = pQueryInfo->order.orderColId;
} else if (functionId == TSDB_FUNC_APERCT) {
pCtx->param[0].i64Key = pExpr->param[0].i64Key;
pCtx->param[0].nType = pExpr->param[0].nType;
} }
pCtx->interBufBytes = pExpr->interBytes; pCtx->interBufBytes = pExpr->interBytes;

1
src/client/src/tscSQLParser.c
View File

@ -5311,6 +5311,7 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) {
pMsg->replications = pCreateDb->replica; pMsg->replications = pCreateDb->replica;
pMsg->quorum = pCreateDb->quorum; pMsg->quorum = pCreateDb->quorum;
pMsg->ignoreExist = pCreateDb->ignoreExists; pMsg->ignoreExist = pCreateDb->ignoreExists;
pMsg->update = pCreateDb->update;
} }
int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) {

24
src/client/src/tscServer.c
View File

@ -182,27 +182,23 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
int32_t waitingDuring = tsShellActivityTimer * 500; int32_t waitingDuring = tsShellActivityTimer * 500;
tscDebug("%p send heartbeat in %dms", pSql, waitingDuring); tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);
taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer); taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer);
} else { } else {
tscDebug("%p start to close tscObj:%p, not send heartbeat again", pSql, pObj); tscDebug("%p start to close tscObj:%p, not send heartbeat again", pSql, pObj);
} }
} }
void tscProcessActivityTimer(void *handle, void *tmrId) { void tscProcessActivityTimer(void *handle, void *tmrId) {
STscObj *pObj = (STscObj *)handle; int64_t rid = (int64_t) handle;
STscObj *pObj = taosAcquireRef(tscRefId, rid);
int ret = taosAcquireRef(tscRefId, pObj); if (pObj == NULL) return;
if (ret < 0) {
tscTrace("%p failed to acquire TSC obj, reason:%s", pObj, tstrerror(ret));
return;
}
SSqlObj* pHB = pObj->pHb; SSqlObj* pHB = pObj->pHb;
void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE)); void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
if (p == NULL) { if (p == NULL) {
tscWarn("%p HB object has been released already", pHB); tscWarn("%p HB object has been released already", pHB);
taosReleaseRef(tscRefId, pObj); taosReleaseRef(tscRefId, pObj->rid);
return; return;
} }
@ -216,7 +212,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) {
tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code)); tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
} }
taosReleaseRef(tscRefId, pObj); taosReleaseRef(tscRefId, rid);
} }
int tscSendMsgToServer(SSqlObj *pSql) { int tscSendMsgToServer(SSqlObj *pSql) {
@ -241,7 +237,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
.pCont = pMsg, .pCont = pMsg,
.contLen = pSql->cmd.payloadLen, .contLen = pSql->cmd.payloadLen,
.ahandle = pSql, .ahandle = pSql,
.handle = &pSql->pRpcCtx, .handle = NULL,
.code = 0 .code = 0
}; };
@ -249,7 +245,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
// Otherwise, the pSql object may have been released already during the response function, which is // Otherwise, the pSql object may have been released already during the response function, which is
// processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
// cause crash. // cause crash.
rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); pSql->rpcRid = rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -269,7 +265,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
assert(*pSql->self == pSql); assert(*pSql->self == pSql);
pSql->pRpcCtx = NULL; pSql->rpcRid = -1;
if (pObj->signature != pObj) { if (pObj->signature != pObj) {
tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature); tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
@ -2026,7 +2022,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
createHBObj(pObj); createHBObj(pObj);
//launch a timer to send heartbeat to maintain the connection and send status to mnode //launch a timer to send heartbeat to maintain the connection and send status to mnode
taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer);
return 0; return 0;
} }

24
src/client/src/tscSql.c
View File

@ -161,7 +161,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
registerSqlObj(pSql); registerSqlObj(pSql);
tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
taosAddRef(tscRefId, pObj); pObj->rid = taosAddRef(tscRefId, pObj);
return pSql; return pSql;
} }
@ -279,9 +279,9 @@ void taos_close(TAOS *taos) {
SSqlObj* pHb = pObj->pHb; SSqlObj* pHb = pObj->pHb;
if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) { if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
if (pHb->pRpcCtx != NULL) { // wait for rsp from dnode if (pHb->rpcRid > 0) { // wait for rsp from dnode
rpcCancelRequest(pHb->pRpcCtx); rpcCancelRequest(pHb->rpcRid);
pHb->pRpcCtx = NULL; pHb->rpcRid = -1;
} }
tscDebug("%p HB is freed", pHb); tscDebug("%p HB is freed", pHb);
@ -298,7 +298,7 @@ void taos_close(TAOS *taos) {
tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
taosRemoveRef(tscRefId, pObj); taosRemoveRef(tscRefId, pObj->rid);
} }
void waitForQueryRsp(void *param, TAOS_RES *tres, int code) { void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
@ -748,9 +748,9 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
assert(pSubObj->self == (SSqlObj**) p); assert(pSubObj->self == (SSqlObj**) p);
pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
if (pSubObj->pRpcCtx != NULL) { if (pSubObj->rpcRid > 0) {
rpcCancelRequest(pSubObj->pRpcCtx); rpcCancelRequest(pSubObj->rpcRid);
pSubObj->pRpcCtx = NULL; pSubObj->rpcRid = -1;
} }
tscQueueAsyncRes(pSubObj); tscQueueAsyncRes(pSubObj);
@ -775,7 +775,7 @@ void taos_stop_query(TAOS_RES *res) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
assert(pSql->pRpcCtx == NULL); assert(pSql->rpcRid <= 0);
tscKillSTableQuery(pSql); tscKillSTableQuery(pSql);
} else { } else {
if (pSql->cmd.command < TSDB_SQL_LOCAL) { if (pSql->cmd.command < TSDB_SQL_LOCAL) {
@ -784,9 +784,9 @@ void taos_stop_query(TAOS_RES *res) {
* reset and freed in the processMsgFromServer function, and causes the invalid * reset and freed in the processMsgFromServer function, and causes the invalid
* write problem for rpcCancelRequest. * write problem for rpcCancelRequest.
*/ */
if (pSql->pRpcCtx != NULL) { if (pSql->rpcRid > 0) {
rpcCancelRequest(pSql->pRpcCtx); rpcCancelRequest(pSql->rpcRid);
pSql->pRpcCtx = NULL; pSql->rpcRid = -1;
} }
tscQueueAsyncRes(pSql); tscQueueAsyncRes(pSql);

4
src/client/src/tscSubquery.c
View File

@ -523,7 +523,7 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
assert(pSqlObj->subState.numOfRemain > 0); assert(pSqlObj->subState.numOfRemain > 0);
if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) { if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) {
tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code); tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code));
freeJoinSubqueryObj(pSqlObj); freeJoinSubqueryObj(pSqlObj);
} }
} }
@ -565,7 +565,7 @@ int32_t tagValCompar(const void* p1, const void* p2) {
return (tag1->len > tag2->len)? 1: -1; return (tag1->len > tag2->len)? 1: -1;
} }
return strncmp(tag1->data, tag2->data, tag1->len); return memcmp(tag1->data, tag2->data, tag1->len);
} }
void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) { void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {

2
src/client/src/tscSystem.c
View File

@ -36,7 +36,7 @@ void * tscTmr;
void * tscQhandle; void * tscQhandle;
void * tscCheckDiskUsageTmr; void * tscCheckDiskUsageTmr;
int tsInsertHeadSize; int tsInsertHeadSize;
int tscRefId; int tscRefId = -1;
int tscNumOfThreads; int tscNumOfThreads;

2
src/client/src/tscUtil.c
View File

@ -376,7 +376,7 @@ void tscFreeRegisteredSqlObj(void *pSql) {
tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref); tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref);
if (ref == 0) { if (ref == 0) {
tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj); tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj);
taosRemoveRef(tscRefId, pTscObj); taosRemoveRef(tscRefId, pTscObj->rid);
} }
} }

84
src/common/inc/tdataformat.h
View File

@ -119,6 +119,33 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
// ----------------- Semantic timestamp key definition
typedef uint64_t TKEY;
#define TKEY_INVALID UINT64_MAX
#define TKEY_NULL TKEY_INVALID
#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
#define TKEY_DELETE_FLAG (((TKEY)1) << 62)
#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0)
#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
if (key1 < key2) {
return -1;
} else if (key1 > key2) {
return 1;
} else {
return 0;
}
}
// ----------------- Data row structure // ----------------- Data row structure
/* A data row, the format is like below: /* A data row, the format is like below:
@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
* +----------+----------+---------------------------------+---------------------------------+ * +----------+----------+---------------------------------+---------------------------------+
* | len | sversion | First part | Second part | * | len | sversion | First part | Second part |
* +----------+----------+---------------------------------+---------------------------------+ * +----------+----------+---------------------------------+---------------------------------+
*
* NOTE: timestamp in this row structure is TKEY instead of TSKEY
*/ */
typedef void *SDataRow; typedef void *SDataRow;
@ -137,11 +166,13 @@ typedef void *SDataRow;
#define dataRowLen(r) (*(uint16_t *)(r)) #define dataRowLen(r) (*(uint16_t *)(r))
#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)) #define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r))) #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
#define dataRowKey(r) tdGetKey(dataRowTKey(r))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l)) #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) #define dataRowSetVersion(r, v) (dataRowVersion(r) = (v))
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r))
SDataRow tdNewDataRowFromSchema(STSchema *pSchema); SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
void tdFreeDataRow(SDataRow row); void tdFreeDataRow(SDataRow row);
@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row)); char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
switch (type) { if (IS_VAR_DATA_TYPE(type)) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
*(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
memcpy(ptr, value, varDataTLen(value)); memcpy(ptr, value, varDataTLen(value));
dataRowLen(row) += varDataTLen(value); dataRowLen(row) += varDataTLen(value);
break; } else {
default: if (offset == 0) {
ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]);
} else {
memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
break; }
} }
return 0; return 0;
@ -171,11 +204,9 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i
// NOTE: offset here including the header size // NOTE: offset here including the header size
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) { static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
switch (type) { if (IS_VAR_DATA_TYPE(type)) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
default: } else {
return POINTER_SHIFT(row, offset); return POINTER_SHIFT(row, offset);
} }
} }
@ -196,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints); void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
void dataColSetOffset(SDataCol *pCol, int nEle); void dataColSetOffset(SDataCol *pCol, int nEle);
bool isNEleNull(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle);
@ -204,27 +234,19 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
// Get the data pointer from a column-wised data // Get the data pointer from a column-wised data
static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
switch (pCol->type) { if (IS_VAR_DATA_TYPE(pCol->type)) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
break; } else {
default:
return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
break;
} }
} }
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
ASSERT(rows > 0); ASSERT(rows > 0);
switch (pDataCol->type) { if (IS_VAR_DATA_TYPE(pDataCol->type)) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
break; } else {
default:
return TYPE_BYTES[pDataCol->type] * rows; return TYPE_BYTES[pDataCol->type] * rows;
} }
} }
@ -243,9 +265,14 @@ typedef struct {
} SDataCols; } SDataCols;
#define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define keyCol(pCols) (&((pCols)->cols[0])) // Key column
#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)] #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]
#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0) #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx))
#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) #define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0))
#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0))
#define dataColsTKeyLast(pCols) \
(((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1))
#define dataColsKeyLast(pCols) \
(((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
void tdResetDataCols(SDataCols *pCols); void tdResetDataCols(SDataCols *pCols);
@ -253,10 +280,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
void tdFreeDataCols(SDataCols *pCols); void tdFreeDataCols(SDataCols *pCols);
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!!
int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows);
// ----------------- K-V data row structure // ----------------- K-V data row structure
/* /*

1
src/common/inc/tglobal.h
View File

@ -88,6 +88,7 @@ extern int16_t tsWAL;
extern int32_t tsFsyncPeriod; extern int32_t tsFsyncPeriod;
extern int32_t tsReplications; extern int32_t tsReplications;
extern int32_t tsQuorum; extern int32_t tsQuorum;
extern int32_t tsUpdate;
// balance // balance
extern int32_t tsEnableBalance; extern int32_t tsEnableBalance;

102
src/common/src/tdataformat.c
View File

@ -18,6 +18,9 @@
#include "tcoding.h" #include "tcoding.h"
#include "wchar.h" #include "wchar.h"
static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows);
/** /**
* Duplicate the schema and return a new object * Duplicate the schema and return a new object
*/ */
@ -202,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE; pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
pDataCol->len = 0; pDataCol->len = 0;
if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { if (IS_VAR_DATA_TYPE(pDataCol->type)) {
pDataCol->dataOff = (VarDataOffsetT *)(*pBuf); pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints); pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
pDataCol->spaceSize = pDataCol->bytes * maxPoints; pDataCol->spaceSize = pDataCol->bytes * maxPoints;
@ -215,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
} }
} }
// value from timestamp should be TKEY here instead of TSKEY
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) { void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
ASSERT(pCol != NULL && value != NULL); ASSERT(pCol != NULL && value != NULL);
switch (pCol->type) { if (IS_VAR_DATA_TYPE(pCol->type)) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
// set offset // set offset
pCol->dataOff[numOfRows] = pCol->len; pCol->dataOff[numOfRows] = pCol->len;
// Copy data // Copy data
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value));
// Update the length // Update the length
pCol->len += varDataTLen(value); pCol->len += varDataTLen(value);
break; } else {
default:
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
pCol->len += pCol->bytes; pCol->len += pCol->bytes;
break;
}
}
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) {
int pointsLeft = numOfRows - pointsToPop;
ASSERT(pointsLeft > 0);
if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) {
ASSERT(pCol->len > 0);
VarDataOffsetT toffset = pCol->dataOff[pointsToPop];
pCol->len = pCol->len - toffset;
ASSERT(pCol->len > 0);
memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len);
dataColSetOffset(pCol, pointsLeft);
} else {
ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
pCol->len = TYPE_BYTES[pCol->type] * pointsLeft;
memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len);
} }
} }
bool isNEleNull(SDataCol *pCol, int nEle) { bool isNEleNull(SDataCol *pCol, int nEle) {
switch (pCol->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
for (int i = 0; i < nEle; i++) { for (int i = 0; i < nEle; i++) {
if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
} }
return true; return true;
default:
for (int i = 0; i < nEle; i++) {
if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
}
return true;
}
} }
void dataColSetNullAt(SDataCol *pCol, int index) { void dataColSetNullAt(SDataCol *pCol, int index) {
@ -390,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize; pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf))); pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
ASSERT(pDataCols->cols[i].dataOff != NULL); ASSERT(pDataCols->cols[i].dataOff != NULL);
pRet->cols[i].dataOff = pRet->cols[i].dataOff =
(int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf))); (int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
@ -400,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
pRet->cols[i].len = pDataCols->cols[i].len; pRet->cols[i].len = pDataCols->cols[i].len;
if (pDataCols->cols[i].len > 0) { if (pDataCols->cols[i].len > 0) {
memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints); memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
} }
} }
@ -420,11 +392,21 @@ void tdResetDataCols(SDataCols *pCols) {
} }
void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
ASSERT(dataColsKeyLast(pCols) < dataRowKey(row)); ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
int rcol = 0; int rcol = 0;
int dcol = 0; int dcol = 0;
if (dataRowDeleted(row)) {
for (; dcol < pCols->numOfCols; dcol++) {
SDataCol *pDataCol = &(pCols->cols[dcol]);
if (dcol == 0) {
dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints);
} else {
dataColSetNullAt(pDataCol, pCols->numOfRows);
}
}
} else {
while (dcol < pCols->numOfCols) { while (dcol < pCols->numOfCols) {
SDataCol *pDataCol = &(pCols->cols[dcol]); SDataCol *pDataCol = &(pCols->cols[dcol]);
if (rcol >= schemaNCols(pSchema)) { if (rcol >= schemaNCols(pSchema)) {
@ -446,32 +428,18 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols)
dcol++; dcol++;
} }
} }
}
pCols->numOfRows++; pCols->numOfRows++;
} }
// Pop pointsToPop points from the SDataCols
void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) {
int pointsLeft = pCols->numOfRows - pointsToPop;
if (pointsLeft <= 0) {
tdResetDataCols(pCols);
return;
}
for (int iCol = 0; iCol < pCols->numOfCols; iCol++) {
SDataCol *pCol = pCols->cols + iCol;
dataColPopPoints(pCol, pointsToPop, pCols->numOfRows);
}
pCols->numOfRows = pointsLeft;
}
int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) {
ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
ASSERT(target->numOfCols == source->numOfCols); ASSERT(target->numOfCols == source->numOfCols);
SDataCols *pTarget = NULL; SDataCols *pTarget = NULL;
if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap
ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
for (int i = 0; i < rowsToMerge; i++) { for (int i = 0; i < rowsToMerge; i++) {
for (int j = 0; j < source->numOfCols; j++) { for (int j = 0; j < source->numOfCols; j++) {
if (source->cols[j].len > 0) { if (source->cols[j].len > 0) {
@ -499,17 +467,23 @@ _err:
return -1; return -1;
} }
void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) { // src2 data has more priority than src1
static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
int limit2, int tRows) {
tdResetDataCols(target); tdResetDataCols(target);
ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
while (target->numOfRows < tRows) { while (target->numOfRows < tRows) {
if (*iter1 >= limit1 && *iter2 >= limit2) break; if (*iter1 >= limit1 && *iter2 >= limit2) break;
TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1]; TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2]; TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
if (key1 <= key2) { ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
if (key1 < key2) {
for (int i = 0; i < src1->numOfCols; i++) { for (int i = 0; i < src1->numOfCols; i++) {
ASSERT(target->cols[i].type == src1->cols[i].type); ASSERT(target->cols[i].type == src1->cols[i].type);
if (src1->cols[i].len > 0) { if (src1->cols[i].len > 0) {
@ -520,8 +494,8 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi
target->numOfRows++; target->numOfRows++;
(*iter1)++; (*iter1)++;
if (key1 == key2) (*iter2)++; } else if (key1 >= key2) {
} else { if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
for (int i = 0; i < src2->numOfCols; i++) { for (int i = 0; i < src2->numOfCols; i++) {
ASSERT(target->cols[i].type == src2->cols[i].type); ASSERT(target->cols[i].type == src2->cols[i].type);
if (src2->cols[i].len > 0) { if (src2->cols[i].len > 0) {
@ -529,10 +503,14 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi
target->maxPoints); target->maxPoints);
} }
} }
target->numOfRows++; target->numOfRows++;
(*iter2)++;
} }
(*iter2)++;
if (key1 == key2) (*iter1)++;
}
ASSERT(target->numOfRows <= target->maxPoints);
} }
} }

11
src/common/src/tglobal.c
View File

@ -120,6 +120,7 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION;
int32_t tsMaxVgroupsPerDb = 0; int32_t tsMaxVgroupsPerDb = 0;
int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP;
int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
@ -786,6 +787,16 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE; cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg); taosInitConfigOption(cfg);
cfg.option = "update";
cfg.ptr = &tsUpdate;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_DB_UPDATE;
cfg.maxValue = TSDB_MAX_DB_UPDATE;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "mqttHostName"; cfg.option = "mqttHostName";
cfg.ptr = tsMqttHostName; cfg.ptr = tsMqttHostName;
cfg.valType = TAOS_CFG_VTYPE_STRING; cfg.valType = TAOS_CFG_VTYPE_STRING;

5
src/inc/taosdef.h
View File

@ -75,6 +75,7 @@ extern const int32_t TYPE_BYTES[11];
#define TSDB_DATA_SMALLINT_NULL 0x8000 #define TSDB_DATA_SMALLINT_NULL 0x8000
#define TSDB_DATA_INT_NULL 0x80000000L #define TSDB_DATA_INT_NULL 0x80000000L
#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L #define TSDB_DATA_BIGINT_NULL 0x8000000000000000L
#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN #define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN #define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN
@ -363,6 +364,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
#define TSDB_MAX_WAL_LEVEL 2 #define TSDB_MAX_WAL_LEVEL 2
#define TSDB_DEFAULT_WAL_LEVEL 1 #define TSDB_DEFAULT_WAL_LEVEL 1
#define TSDB_MIN_DB_UPDATE 0
#define TSDB_MAX_DB_UPDATE 1
#define TSDB_DEFAULT_DB_UPDATE_OPTION 0
#define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second

5
src/inc/taosmsg.h
View File

@ -544,6 +544,8 @@ typedef struct {
int8_t replications; int8_t replications;
int8_t quorum; int8_t quorum;
int8_t ignoreExist; int8_t ignoreExist;
int8_t update;
int8_t reserve[9];
} SCreateDbMsg, SAlterDbMsg; } SCreateDbMsg, SAlterDbMsg;
typedef struct { typedef struct {
@ -654,7 +656,8 @@ typedef struct {
int8_t replications; int8_t replications;
int8_t wals; int8_t wals;
int8_t quorum; int8_t quorum;
int8_t reserved[16]; int8_t update;
int8_t reserved[15];
} SVnodeCfg; } SVnodeCfg;
typedef struct { typedef struct {

4
src/inc/trpc.h
View File

@ -83,13 +83,13 @@ void rpcClose(void *);
void *rpcMallocCont(int contLen); void *rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont); void rpcFreeCont(void *pCont);
void *rpcReallocCont(void *ptr, int contLen); void *rpcReallocCont(void *ptr, int contLen);
void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg); int64_t rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg);
void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet); void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen); int rpcReportProgress(void *pConn, char *pCont, int contLen);
void rpcCancelRequest(void *pContext); void rpcCancelRequest(int64_t rid);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
src/inc/tsdb.h
View File

@ -65,6 +65,7 @@ typedef struct {
int32_t maxRowsPerFileBlock; // maximum rows per file block int32_t maxRowsPerFileBlock; // maximum rows per file block
int8_t precision; int8_t precision;
int8_t compression; int8_t compression;
int8_t update;
} STsdbCfg; } STsdbCfg;
// --------- TSDB REPOSITORY USAGE STATISTICS // --------- TSDB REPOSITORY USAGE STATISTICS

14
src/inc/tsync.h
View File

@ -106,13 +106,13 @@ typedef void* tsync_h;
int32_t syncInit(); int32_t syncInit();
void syncCleanUp(); void syncCleanUp();
tsync_h syncStart(const SSyncInfo *); int64_t syncStart(const SSyncInfo *);
void syncStop(tsync_h shandle); void syncStop(int64_t rid);
int32_t syncReconfig(tsync_h shandle, const SSyncCfg *); int32_t syncReconfig(int64_t rid, const SSyncCfg *);
int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype); int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype);
void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code); void syncConfirmForward(int64_t rid, uint64_t version, int32_t code);
void syncRecover(tsync_h shandle); // recover from other nodes: void syncRecover(int64_t rid); // recover from other nodes:
int syncGetNodesRole(tsync_h shandle, SNodesRole *); int syncGetNodesRole(int64_t rid, SNodesRole *);
extern char *syncRole[]; extern char *syncRole[];

217
src/inc/ttokendef.h
View File

@ -114,114 +114,115 @@
#define TK_FSYNC 95 #define TK_FSYNC 95
#define TK_COMP 96 #define TK_COMP 96
#define TK_PRECISION 97 #define TK_PRECISION 97
#define TK_LP 98 #define TK_UPDATE 98
#define TK_RP 99 #define TK_LP 99
#define TK_TAGS 100 #define TK_RP 100
#define TK_USING 101 #define TK_TAGS 101
#define TK_AS 102 #define TK_USING 102
#define TK_COMMA 103 #define TK_AS 103
#define TK_NULL 104 #define TK_COMMA 104
#define TK_SELECT 105 #define TK_NULL 105
#define TK_UNION 106 #define TK_SELECT 106
#define TK_ALL 107 #define TK_UNION 107
#define TK_FROM 108 #define TK_ALL 108
#define TK_VARIABLE 109 #define TK_FROM 109
#define TK_INTERVAL 110 #define TK_VARIABLE 110
#define TK_FILL 111 #define TK_INTERVAL 111
#define TK_SLIDING 112 #define TK_FILL 112
#define TK_ORDER 113 #define TK_SLIDING 113
#define TK_BY 114 #define TK_ORDER 114
#define TK_ASC 115 #define TK_BY 115
#define TK_DESC 116 #define TK_ASC 116
#define TK_GROUP 117 #define TK_DESC 117
#define TK_HAVING 118 #define TK_GROUP 118
#define TK_LIMIT 119 #define TK_HAVING 119
#define TK_OFFSET 120 #define TK_LIMIT 120
#define TK_SLIMIT 121 #define TK_OFFSET 121
#define TK_SOFFSET 122 #define TK_SLIMIT 122
#define TK_WHERE 123 #define TK_SOFFSET 123
#define TK_NOW 124 #define TK_WHERE 124
#define TK_RESET 125 #define TK_NOW 125
#define TK_QUERY 126 #define TK_RESET 126
#define TK_ADD 127 #define TK_QUERY 127
#define TK_COLUMN 128 #define TK_ADD 128
#define TK_TAG 129 #define TK_COLUMN 129
#define TK_CHANGE 130 #define TK_TAG 130
#define TK_SET 131 #define TK_CHANGE 131
#define TK_KILL 132 #define TK_SET 132
#define TK_CONNECTION 133 #define TK_KILL 133
#define TK_STREAM 134 #define TK_CONNECTION 134
#define TK_COLON 135 #define TK_STREAM 135
#define TK_ABORT 136 #define TK_COLON 136
#define TK_AFTER 137 #define TK_ABORT 137
#define TK_ATTACH 138 #define TK_AFTER 138
#define TK_BEFORE 139 #define TK_ATTACH 139
#define TK_BEGIN 140 #define TK_BEFORE 140
#define TK_CASCADE 141 #define TK_BEGIN 141
#define TK_CLUSTER 142 #define TK_CASCADE 142
#define TK_CONFLICT 143 #define TK_CLUSTER 143
#define TK_COPY 144 #define TK_CONFLICT 144
#define TK_DEFERRED 145 #define TK_COPY 145
#define TK_DELIMITERS 146 #define TK_DEFERRED 146
#define TK_DETACH 147 #define TK_DELIMITERS 147
#define TK_EACH 148 #define TK_DETACH 148
#define TK_END 149 #define TK_EACH 149
#define TK_EXPLAIN 150 #define TK_END 150
#define TK_FAIL 151 #define TK_EXPLAIN 151
#define TK_FOR 152 #define TK_FAIL 152
#define TK_IGNORE 153 #define TK_FOR 153
#define TK_IMMEDIATE 154 #define TK_IGNORE 154
#define TK_INITIALLY 155 #define TK_IMMEDIATE 155
#define TK_INSTEAD 156 #define TK_INITIALLY 156
#define TK_MATCH 157 #define TK_INSTEAD 157
#define TK_KEY 158 #define TK_MATCH 158
#define TK_OF 159 #define TK_KEY 159
#define TK_RAISE 160 #define TK_OF 160
#define TK_REPLACE 161 #define TK_RAISE 161
#define TK_RESTRICT 162 #define TK_REPLACE 162
#define TK_ROW 163 #define TK_RESTRICT 163
#define TK_STATEMENT 164 #define TK_ROW 164
#define TK_TRIGGER 165 #define TK_STATEMENT 165
#define TK_VIEW 166 #define TK_TRIGGER 166
#define TK_COUNT 167 #define TK_VIEW 167
#define TK_SUM 168 #define TK_COUNT 168
#define TK_AVG 169 #define TK_SUM 169
#define TK_MIN 170 #define TK_AVG 170
#define TK_MAX 171 #define TK_MIN 171
#define TK_FIRST 172 #define TK_MAX 172
#define TK_LAST 173 #define TK_FIRST 173
#define TK_TOP 174 #define TK_LAST 174
#define TK_BOTTOM 175 #define TK_TOP 175
#define TK_STDDEV 176 #define TK_BOTTOM 176
#define TK_PERCENTILE 177 #define TK_STDDEV 177
#define TK_APERCENTILE 178 #define TK_PERCENTILE 178
#define TK_LEASTSQUARES 179 #define TK_APERCENTILE 179
#define TK_HISTOGRAM 180 #define TK_LEASTSQUARES 180
#define TK_DIFF 181 #define TK_HISTOGRAM 181
#define TK_SPREAD 182 #define TK_DIFF 182
#define TK_TWA 183 #define TK_SPREAD 183
#define TK_INTERP 184 #define TK_TWA 184
#define TK_LAST_ROW 185 #define TK_INTERP 185
#define TK_RATE 186 #define TK_LAST_ROW 186
#define TK_IRATE 187 #define TK_RATE 187
#define TK_SUM_RATE 188 #define TK_IRATE 188
#define TK_SUM_IRATE 189 #define TK_SUM_RATE 189
#define TK_AVG_RATE 190 #define TK_SUM_IRATE 190
#define TK_AVG_IRATE 191 #define TK_AVG_RATE 191
#define TK_TBID 192 #define TK_AVG_IRATE 192
#define TK_SEMI 193 #define TK_TBID 193
#define TK_NONE 194 #define TK_SEMI 194
#define TK_PREV 195 #define TK_NONE 195
#define TK_LINEAR 196 #define TK_PREV 196
#define TK_IMPORT 197 #define TK_LINEAR 197
#define TK_METRIC 198 #define TK_IMPORT 198
#define TK_TBNAME 199 #define TK_METRIC 199
#define TK_JOIN 200 #define TK_TBNAME 200
#define TK_METRICS 201 #define TK_JOIN 201
#define TK_STABLE 202 #define TK_METRICS 202
#define TK_INSERT 203 #define TK_STABLE 203
#define TK_INTO 204 #define TK_INSERT 204
#define TK_VALUES 205 #define TK_INTO 205
#define TK_VALUES 206
#define TK_SPACE 300 #define TK_SPACE 300
#define TK_COMMENT 301 #define TK_COMMENT 301

3
src/mnode/inc/mnodeDef.h
View File

@ -172,7 +172,8 @@ typedef struct {
int8_t walLevel; int8_t walLevel;
int8_t replications; int8_t replications;
int8_t quorum; int8_t quorum;
int8_t reserved[12]; int8_t update;
int8_t reserved[11];
} SDbCfg; } SDbCfg;
typedef struct SDbObj { typedef struct SDbObj {

30
src/mnode/src/mnodeDb.c
View File

@ -319,6 +319,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
} }
#endif #endif
if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE);
return TSDB_CODE_MND_INVALID_DB_OPTION;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -339,6 +344,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL;
if (pCfg->replications < 0) pCfg->replications = tsReplications; if (pCfg->replications < 0) pCfg->replications = tsReplications;
if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; if (pCfg->quorum < 0) pCfg->quorum = tsQuorum;
if (pCfg->update < 0) pCfg->update = tsUpdate;
} }
static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
@ -391,7 +397,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *
.compression = pCreate->compression, .compression = pCreate->compression,
.walLevel = pCreate->walLevel, .walLevel = pCreate->walLevel,
.replications = pCreate->replications, .replications = pCreate->replications,
.quorum = pCreate->quorum .quorum = pCreate->quorum,
.update = pCreate->update
}; };
mnodeSetDefaultDbCfg(&pDb->cfg); mnodeSetDefaultDbCfg(&pDb->cfg);
@ -610,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
pSchema[cols].bytes = htons(pShow->bytes[cols]); pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++; cols++;
pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema[cols].name, "update");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status"); strcpy(pSchema[cols].name, "status");
@ -749,6 +762,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void
STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
cols++; cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int8_t *)pWrite = pDb->cfg.update;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
if (pDb->status == TSDB_DB_STATUS_READY) { if (pDb->status == TSDB_DB_STATUS_READY) {
const char *src = "ready"; const char *src = "ready";
@ -848,6 +865,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
int8_t replications = pAlter->replications; int8_t replications = pAlter->replications;
int8_t quorum = pAlter->quorum; int8_t quorum = pAlter->quorum;
int8_t precision = pAlter->precision; int8_t precision = pAlter->precision;
int8_t update = pAlter->update;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
@ -950,6 +968,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
newCfg.quorum = quorum; newCfg.quorum = quorum;
} }
if (update >= 0 && update != pDb->cfg.update) {
#if 0
mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update);
newCfg.update = update;
#else
mError("db:%s, can't alter update option", pDb->name);
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
#endif
}
return newCfg; return newCfg;
} }

6
src/mnode/src/mnodeSdb.c
View File

@ -72,7 +72,7 @@ typedef struct {
ESyncRole role; ESyncRole role;
ESdbStatus status; ESdbStatus status;
int64_t version; int64_t version;
void * sync; int64_t sync;
void * wal; void * wal;
SSyncCfg cfg; SSyncCfg cfg;
int32_t numOfTables; int32_t numOfTables;
@ -212,7 +212,7 @@ static void sdbRestoreTables() {
} }
void sdbUpdateMnodeRoles() { void sdbUpdateMnodeRoles() {
if (tsSdbObj.sync == NULL) return; if (tsSdbObj.sync <= 0) return;
SNodesRole roles = {0}; SNodesRole roles = {0};
syncGetNodesRole(tsSdbObj.sync, &roles); syncGetNodesRole(tsSdbObj.sync, &roles);
@ -433,7 +433,7 @@ void sdbCleanUp() {
if (tsSdbObj.sync) { if (tsSdbObj.sync) {
syncStop(tsSdbObj.sync); syncStop(tsSdbObj.sync);
tsSdbObj.sync = NULL; tsSdbObj.sync = -1;
} }
if (tsSdbObj.wal) { if (tsSdbObj.wal) {

1
src/mnode/src/mnodeVgroup.c
View File

@ -850,6 +850,7 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) {
pCfg->replications = (int8_t) pVgroup->numOfVnodes; pCfg->replications = (int8_t) pVgroup->numOfVnodes;
pCfg->wals = 3; pCfg->wals = 3;
pCfg->quorum = pDb->cfg.quorum; pCfg->quorum = pDb->cfg.quorum;
pCfg->update = pDb->cfg.update;
SVnodeDesc *pNodes = pVnode->nodes; SVnodeDesc *pNodes = pVnode->nodes;
for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) {

1
src/query/inc/qSqlparser.h
View File

@ -129,6 +129,7 @@ typedef struct SCreateDBInfo {
int32_t compressionLevel; int32_t compressionLevel;
SStrToken precision; SStrToken precision;
bool ignoreExists; bool ignoreExists;
int8_t update;
tVariantList *keep; tVariantList *keep;
} SCreateDBInfo; } SCreateDBInfo;

3
src/query/inc/sql.y
View File

@ -239,6 +239,7 @@ wal(Y) ::= WAL INTEGER(X). { Y = X; }
fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } fsync(Y) ::= FSYNC INTEGER(X). { Y = X; }
comp(Y) ::= COMP INTEGER(X). { Y = X; } comp(Y) ::= COMP INTEGER(X). { Y = X; }
prec(Y) ::= PRECISION STRING(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; }
update(Y) ::= UPDATE INTEGER(X). { Y = X; }
%type db_optr {SCreateDBInfo} %type db_optr {SCreateDBInfo}
db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);}
@ -256,6 +257,7 @@ db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z
db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; }
db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; }
db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type alter_db_optr {SCreateDBInfo} %type alter_db_optr {SCreateDBInfo}
alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);}
@ -267,6 +269,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s
alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }
alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); }
%type typename {TAOS_FIELD} %type typename {TAOS_FIELD}
typename(A) ::= ids(X). { typename(A) ::= ids(X). {

2
src/query/inc/tsqlfunction.h
View File

@ -149,7 +149,7 @@ typedef struct SResultRowCellInfo {
int8_t hasResult; // result generated, not NULL value int8_t hasResult; // result generated, not NULL value
bool initialized; // output buffer has been initialized bool initialized; // output buffer has been initialized
bool complete; // query has completed bool complete; // query has completed
uint16_t numOfRes; // num of output result in current buffer uint32_t numOfRes; // num of output result in current buffer
} SResultRowCellInfo; } SResultRowCellInfo;
#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) #define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))

25
src/query/src/qAst.c
View File

@ -32,7 +32,6 @@
#include "tstoken.h" #include "tstoken.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "tulog.h" #include "tulog.h"
#include "tutil.h"
/* /*
* *
@ -415,9 +414,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
} }
if (cond.start != NULL) { if (cond.start != NULL) {
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC); iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC);
} else { } else {
iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC); iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC);
} }
if (cond.start != NULL) { if (cond.start != NULL) {
@ -432,7 +431,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
break; break;
} }
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info); taosArrayPush(result, &info);
} }
} else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal
@ -450,7 +449,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_GREATER) { if (ret == 0 && optr == TSDB_RELATION_GREATER) {
continue; continue;
} else { } else {
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info); taosArrayPush(result, &info);
comp = false; comp = false;
} }
@ -465,14 +464,14 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
continue; continue;
} }
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info); taosArrayPush(result, &info);
} }
tSkipListDestroyIter(iter); tSkipListDestroyIter(iter);
comp = true; comp = true;
iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC); iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC);
while(tSkipListIterNext(iter)) { while(tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter); SSkipListNode* pNode = tSkipListIterGet(iter);
comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0);
@ -480,7 +479,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
continue; continue;
} }
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info); taosArrayPush(result, &info);
} }
@ -504,7 +503,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
if (ret == 0 && optr == TSDB_RELATION_LESS) { if (ret == 0 && optr == TSDB_RELATION_LESS) {
continue; continue;
} else { } else {
STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info); taosArrayPush(result, &info);
comp = false; // no need to compare anymore comp = false; // no need to compare anymore
} }
@ -518,7 +517,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type);
if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) ||
(pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) {
STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info); taosArrayPush(result, &info);
} }
} }
@ -682,7 +681,7 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki
while (tSkipListIterNext(iter)) { while (tSkipListIterNext(iter)) {
SSkipListNode *pNode = tSkipListIterGet(iter); SSkipListNode *pNode = tSkipListIterGet(iter);
if (filterItem(pExpr, pNode, param)) { if (filterItem(pExpr, pNode, param)) {
taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode)));
} }
} }
tSkipListDestroyIter(iter); tSkipListDestroyIter(iter);
@ -697,7 +696,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
SSkipListNode *pNode = tSkipListIterGet(iter); SSkipListNode *pNode = tSkipListIterGet(iter);
char * pData = SL_GET_NODE_DATA(pNode); char * pData = SL_GET_NODE_DATA(pNode);
tstr *name = (tstr*) tsdbGetTableName(*(void**) pData); tstr *name = (tstr*) tsdbGetTableName((void*) pData);
// todo speed up by using hash // todo speed up by using hash
if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
@ -711,7 +710,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
} }
if (addToResult) { if (addToResult) {
STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL}; STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(res, &info); taosArrayPush(res, &info);
} }
} }

3
src/query/src/qExecutor.c
View File

@ -1609,7 +1609,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx));
pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t));
pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);//calloc(1, sizeof(SResultRow)); pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);
if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) {
goto _clean; goto _clean;
@ -1745,6 +1745,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf);
taosTFree(pRuntimeEnv->keyBuf); taosTFree(pRuntimeEnv->keyBuf);
taosTFree(pRuntimeEnv->rowCellInfoOffset);
taosHashCleanup(pRuntimeEnv->pResultRowHashTable); taosHashCleanup(pRuntimeEnv->pResultRowHashTable);
pRuntimeEnv->pResultRowHashTable = NULL; pRuntimeEnv->pResultRowHashTable = NULL;

2
src/query/src/qFill.c
View File

@ -170,7 +170,7 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
int32_t numOfRows = taosNumOfRemainRows(pFillInfo); int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey; TSKEY ekey1 = ekey;
if (pFillInfo->order != TSDB_ORDER_ASC) { if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
} }

2
src/query/src/qHistogram.c
View File

@ -168,7 +168,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
(*pHisto)->numOfEntries += 1; (*pHisto)->numOfEntries += 1;
} }
} else { /* insert a new slot */ } else { /* insert a new slot */
if ((*pHisto)->numOfElems > 1 && idx < (*pHisto)->numOfEntries) { if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) {
if (idx > 0) { if (idx > 0) {
assert((*pHisto)->elems[idx - 1].val <= val); assert((*pHisto)->elems[idx - 1].val <= val);
} }

1
src/query/src/qParserImpl.c
View File

@ -872,5 +872,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
pDBInfo->quorum = -1; pDBInfo->quorum = -1;
pDBInfo->keep = NULL; pDBInfo->keep = NULL;
pDBInfo->update = -1;
memset(&pDBInfo->precision, 0, sizeof(SStrToken)); memset(&pDBInfo->precision, 0, sizeof(SStrToken));
} }

1
src/query/src/qTokenizer.c
View File

@ -155,6 +155,7 @@ static SKeyword keywordTable[] = {
{"INSERT", TK_INSERT}, {"INSERT", TK_INSERT},
{"INTO", TK_INTO}, {"INTO", TK_INTO},
{"VALUES", TK_VALUES}, {"VALUES", TK_VALUES},
{"UPDATE", TK_UPDATE},
{"RESET", TK_RESET}, {"RESET", TK_RESET},
{"QUERY", TK_QUERY}, {"QUERY", TK_QUERY},
{"ADD", TK_ADD}, {"ADD", TK_ADD},

2328
src/query/src/sql.c
View File

File diff suppressed because it is too large Load Diff

35
src/rpc/src/rpcMain.c
View File

@ -82,6 +82,7 @@ typedef struct {
int8_t oldInUse; // server EP inUse passed by app int8_t oldInUse; // server EP inUse passed by app
int8_t redirect; // flag to indicate redirect int8_t redirect; // flag to indicate redirect
int8_t connType; // connection type int8_t connType; // connection type
int64_t rid; // refId returned by taosAddRef
SRpcMsg *pRsp; // for synchronous API SRpcMsg *pRsp; // for synchronous API
tsem_t *pSem; // for synchronous API tsem_t *pSem; // for synchronous API
SRpcEpSet *pSet; // for synchronous API SRpcEpSet *pSet; // for synchronous API
@ -220,8 +221,7 @@ static void rpcFree(void *p) {
free(p); free(p);
} }
static void rpcInit(void) { void rpcInit(void) {
tsProgressTimer = tsRpcTimer/2; tsProgressTimer = tsRpcTimer/2;
tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer; tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer;
tsRpcHeadSize = RPC_MSG_OVERHEAD; tsRpcHeadSize = RPC_MSG_OVERHEAD;
@ -230,6 +230,11 @@ static void rpcInit(void) {
tsRpcRefId = taosOpenRef(200, rpcFree); tsRpcRefId = taosOpenRef(200, rpcFree);
} }
void rpcCleanup(void) {
taosCloseRef(tsRpcRefId);
tsRpcRefId = -1;
}
void *rpcOpen(const SRpcInit *pInit) { void *rpcOpen(const SRpcInit *pInit) {
SRpcInfo *pRpc; SRpcInfo *pRpc;
@ -374,7 +379,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
} }
void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcInfo *pRpc = (SRpcInfo *)shandle;
SRpcReqContext *pContext; SRpcReqContext *pContext;
@ -403,10 +408,11 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
// set the handle to pContext, so app can cancel the request // set the handle to pContext, so app can cancel the request
if (pMsg->handle) *((void **)pMsg->handle) = pContext; if (pMsg->handle) *((void **)pMsg->handle) = pContext;
taosAddRef(tsRpcRefId, pContext); pContext->rid = taosAddRef(tsRpcRefId, pContext);
rpcSendReqToServer(pRpc, pContext); rpcSendReqToServer(pRpc, pContext);
return; return pContext->rid;
} }
void rpcSendResponse(const SRpcMsg *pRsp) { void rpcSendResponse(const SRpcMsg *pRsp) {
@ -551,15 +557,14 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) {
return code; return code;
} }
void rpcCancelRequest(void *handle) { void rpcCancelRequest(int64_t rid) {
SRpcReqContext *pContext = handle;
int code = taosAcquireRef(tsRpcRefId, pContext); SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid);
if (code < 0) return; if (pContext == NULL) return;
rpcCloseConn(pContext->pConn); rpcCloseConn(pContext->pConn);
taosReleaseRef(tsRpcRefId, pContext); taosReleaseRef(tsRpcRefId, rid);
} }
static void rpcFreeMsg(void *msg) { static void rpcFreeMsg(void *msg) {
@ -628,7 +633,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
// if there is an outgoing message, free it // if there is an outgoing message, free it
if (pConn->outType && pConn->pReqMsg) { if (pConn->outType && pConn->pReqMsg) {
if (pConn->pContext) pConn->pContext->pConn = NULL; if (pConn->pContext) pConn->pContext->pConn = NULL;
taosRemoveRef(tsRpcRefId, pConn->pContext); taosRemoveRef(tsRpcRefId, pConn->pContext->rid);
} }
} }
@ -1109,7 +1114,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
} }
// free the request message // free the request message
taosRemoveRef(tsRpcRefId, pContext); taosRemoveRef(tsRpcRefId, pContext->rid);
} }
static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) { static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) {
@ -1620,11 +1625,7 @@ static void rpcDecRef(SRpcInfo *pRpc)
tDebug("%s rpc resources are released", pRpc->label); tDebug("%s rpc resources are released", pRpc->label);
taosTFree(pRpc); taosTFree(pRpc);
int count = atomic_sub_fetch_32(&tsRpcNum, 1); atomic_sub_fetch_32(&tsRpcNum, 1);
if (count == 0) {
// taosCloseRef(tsRpcRefId);
// tsRpcInit = PTHREAD_ONCE_INIT; // windows compliling error
}
} }
} }

1
src/sync/inc/syncInt.h
View File

@ -141,6 +141,7 @@ typedef struct SSyncNode {
int8_t replica; int8_t replica;
int8_t quorum; int8_t quorum;
uint32_t vgId; uint32_t vgId;
int64_t rid;
void *ahandle; void *ahandle;
int8_t selfIndex; int8_t selfIndex;
SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator

104
src/sync/src/syncMain.c
View File

@ -142,14 +142,14 @@ void syncCleanUp() {
sInfo("sync module is cleaned up"); sInfo("sync module is cleaned up");
} }
void *syncStart(const SSyncInfo *pInfo) { int64_t syncStart(const SSyncInfo *pInfo) {
const SSyncCfg *pCfg = &pInfo->syncCfg; const SSyncCfg *pCfg = &pInfo->syncCfg;
SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1); SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1);
if (pNode == NULL) { if (pNode == NULL) {
sError("no memory to allocate syncNode"); sError("no memory to allocate syncNode");
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return NULL; return -1;
} }
tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path)); tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
@ -170,10 +170,10 @@ void *syncStart(const SSyncInfo *pInfo) {
pNode->quorum = pCfg->quorum; pNode->quorum = pCfg->quorum;
if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
int ret = taosAddRef(tsSyncRefId, pNode); pNode->rid = taosAddRef(tsSyncRefId, pNode);
if (ret < 0) { if (pNode->rid < 0) {
syncFreeNode(pNode); syncFreeNode(pNode);
return NULL; return -1;
} }
for (int i = 0; i < pCfg->replica; ++i) { for (int i = 0; i < pCfg->replica; ++i) {
@ -187,8 +187,8 @@ void *syncStart(const SSyncInfo *pInfo) {
if (pNode->selfIndex < 0) { if (pNode->selfIndex < 0) {
sInfo("vgId:%d, this node is not configured", pNode->vgId); sInfo("vgId:%d, this node is not configured", pNode->vgId);
terrno = TSDB_CODE_SYN_INVALID_CONFIG; terrno = TSDB_CODE_SYN_INVALID_CONFIG;
syncStop(pNode); syncStop(pNode->rid);
return NULL; return -1;
} }
nodeVersion = pInfo->version; // set the initial version nodeVersion = pInfo->version; // set the initial version
@ -200,15 +200,15 @@ void *syncStart(const SSyncInfo *pInfo) {
if (pNode->pSyncFwds == NULL) { if (pNode->pSyncFwds == NULL) {
sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId); sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
syncStop(pNode); syncStop(pNode->rid);
return NULL; return -1;
} }
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl);
if (pNode->pFwdTimer == NULL) { if (pNode->pFwdTimer == NULL) {
sError("vgId:%d, failed to allocate timer", pNode->vgId); sError("vgId:%d, failed to allocate timer", pNode->vgId);
syncStop(pNode); syncStop(pNode->rid);
return NULL; return -1;
} }
syncAddArbitrator(pNode); syncAddArbitrator(pNode);
@ -218,15 +218,14 @@ void *syncStart(const SSyncInfo *pInfo) {
(*pNode->notifyRole)(pNode->ahandle, nodeRole); (*pNode->notifyRole)(pNode->ahandle, nodeRole);
} }
return pNode; return pNode->rid;
} }
void syncStop(void *param) { void syncStop(int64_t rid) {
SSyncNode *pNode = param;
SSyncPeer *pPeer; SSyncPeer *pPeer;
int ret = taosAcquireRef(tsSyncRefId, pNode); SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (ret < 0) return; if (pNode == NULL) return;
sInfo("vgId:%d, cleanup sync", pNode->vgId); sInfo("vgId:%d, cleanup sync", pNode->vgId);
@ -245,16 +244,15 @@ void syncStop(void *param) {
pthread_mutex_unlock(&(pNode->mutex)); pthread_mutex_unlock(&(pNode->mutex));
taosReleaseRef(tsSyncRefId, pNode); taosReleaseRef(tsSyncRefId, rid);
taosRemoveRef(tsSyncRefId, pNode); taosRemoveRef(tsSyncRefId, rid);
} }
int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
SSyncNode *pNode = param;
int i, j; int i, j;
int ret = taosAcquireRef(tsSyncRefId, pNode); SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (ret < 0) return TSDB_CODE_SYN_INVALID_CONFIG; if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
pNode->replica); pNode->replica);
@ -318,29 +316,25 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
syncRole[nodeRole]); syncRole[nodeRole]);
syncBroadcastStatus(pNode); syncBroadcastStatus(pNode);
taosReleaseRef(tsSyncRefId, pNode); taosReleaseRef(tsSyncRefId, rid);
return 0; return 0;
} }
int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) { int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int qtype) {
SSyncNode *pNode = param; SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return 0;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if (ret < 0) return 0;
int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype);
taosReleaseRef(tsSyncRefId, pNode); taosReleaseRef(tsSyncRefId, rid);
return code; return code;
} }
void syncConfirmForward(void *param, uint64_t version, int32_t code) { void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
SSyncNode *pNode = param; SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if (ret < 0) return;
SSyncPeer *pPeer = pNode->pMaster; SSyncPeer *pPeer = pNode->pMaster;
if (pPeer && pNode->quorum > 1) { if (pPeer && pNode->quorum > 1) {
@ -365,15 +359,14 @@ void syncConfirmForward(void *param, uint64_t version, int32_t code) {
} }
} }
taosReleaseRef(tsSyncRefId, pNode); taosReleaseRef(tsSyncRefId, rid);
} }
void syncRecover(void *param) { void syncRecover(int64_t rid) {
SSyncNode *pNode = param;
SSyncPeer *pPeer; SSyncPeer *pPeer;
int ret = taosAcquireRef(tsSyncRefId, pNode); SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (ret < 0) return; if (pNode == NULL) return;
// to do: add a few lines to check if recover is OK // to do: add a few lines to check if recover is OK
// if take this node to unsync state, the whole system may not work // if take this node to unsync state, the whole system may not work
@ -393,14 +386,12 @@ void syncRecover(void *param) {
pthread_mutex_unlock(&(pNode->mutex)); pthread_mutex_unlock(&(pNode->mutex));
taosReleaseRef(tsSyncRefId, pNode); taosReleaseRef(tsSyncRefId, rid);
} }
int syncGetNodesRole(void *param, SNodesRole *pNodesRole) { int syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
SSyncNode *pNode = param; SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
if (pNode == NULL) return -1;
int ret = taosAcquireRef(tsSyncRefId, pNode);
if (ret < 0) return -1;
pNodesRole->selfIndex = pNode->selfIndex; pNodesRole->selfIndex = pNode->selfIndex;
for (int i = 0; i < pNode->replica; ++i) { for (int i = 0; i < pNode->replica; ++i) {
@ -408,7 +399,7 @@ int syncGetNodesRole(void *param, SNodesRole *pNodesRole) {
pNodesRole->role[i] = pNode->peerInfo[i]->role; pNodesRole->role[i] = pNode->peerInfo[i]->role;
} }
taosReleaseRef(tsSyncRefId, pNode); taosReleaseRef(tsSyncRefId, rid);
return 0; return 0;
} }
@ -455,7 +446,7 @@ void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1);
int syncDecPeerRef(SSyncPeer *pPeer) { int syncDecPeerRef(SSyncPeer *pPeer) {
if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
taosReleaseRef(tsSyncRefId, pPeer->pSyncNode); taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
sDebug("%s, resource is freed", pPeer->id); sDebug("%s, resource is freed", pPeer->id);
taosTFree(pPeer->watchFd); taosTFree(pPeer->watchFd);
@ -512,7 +503,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer); taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer);
} }
taosAcquireRef(tsSyncRefId, pNode); taosAcquireRef(tsSyncRefId, pNode->rid);
return pPeer; return pPeer;
} }
@ -1105,7 +1096,7 @@ static void syncProcessBrokenLink(void *param) {
SSyncPeer *pPeer = param; SSyncPeer *pPeer = param;
SSyncNode *pNode = pPeer->pSyncNode; SSyncNode *pNode = pPeer->pSyncNode;
if (taosAcquireRef(tsSyncRefId, pNode) < 0) return; if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return;
pthread_mutex_lock(&(pNode->mutex)); pthread_mutex_lock(&(pNode->mutex));
sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno)); sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
@ -1116,7 +1107,7 @@ static void syncProcessBrokenLink(void *param) {
} }
pthread_mutex_unlock(&(pNode->mutex)); pthread_mutex_unlock(&(pNode->mutex));
taosReleaseRef(tsSyncRefId, pNode); taosReleaseRef(tsSyncRefId, pNode->rid);
} }
static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
@ -1184,10 +1175,9 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
} }
static void syncMonitorFwdInfos(void *param, void *tmrId) { static void syncMonitorFwdInfos(void *param, void *tmrId) {
SSyncNode *pNode = param; int64_t rid = (int64_t) param;
SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
int ret = taosAcquireRef(tsSyncRefId, pNode); if (pNode == NULL) return;
if ( ret < 0) return;
SSyncFwds *pSyncFwds = pNode->pSyncFwds; SSyncFwds *pSyncFwds = pNode->pSyncFwds;
@ -1206,10 +1196,10 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
pthread_mutex_unlock(&(pNode->mutex)); pthread_mutex_unlock(&(pNode->mutex));
} }
pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl);
} }
taosReleaseRef(tsSyncRefId, pNode); taosReleaseRef(tsSyncRefId, rid);
} }
static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtype) { static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtype) {

2
src/sync/test/syncServer.c
View File

@ -30,7 +30,7 @@ int dataFd = -1;
void * qhandle = NULL; void * qhandle = NULL;
int walNum = 0; int walNum = 0;
uint64_t tversion = 0; uint64_t tversion = 0;
void * syncHandle; int64_t syncHandle;
int role; int role;
int nodeId; int nodeId;
char path[256]; char path[256];

24
src/tsdb/inc/tsdbMain.h
View File

@ -320,6 +320,15 @@ typedef struct {
void* compBuffer; // Buffer for temperary compress/decompress purpose void* compBuffer; // Buffer for temperary compress/decompress purpose
} SRWHelper; } SRWHelper;
typedef struct {
int rowsInserted;
int rowsUpdated;
int rowsDeleteSucceed;
int rowsDeleteFailed;
int nOperations;
TSKEY keyFirst;
TSKEY keyLast;
} SMergeInfo;
// ------------------ tsdbScan.c // ------------------ tsdbScan.c
typedef struct { typedef struct {
SFileGroup fGroup; SFileGroup fGroup;
@ -422,7 +431,7 @@ void tsdbCloseBufPool(STsdbRepo* pRepo);
SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
// ------------------ tsdbMemTable.c // ------------------ tsdbMemTable.c
int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); int tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
@ -430,7 +439,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem)
void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbAsyncCommit(STsdbRepo* pRepo);
int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
TSKEY* filterKeys, int nFilterKeys); TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
if (pIter == NULL) return NULL; if (pIter == NULL) return NULL;
@ -438,16 +447,23 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
SSkipListNode* node = tSkipListIterGet(pIter); SSkipListNode* node = tSkipListIterGet(pIter);
if (node == NULL) return NULL; if (node == NULL) return NULL;
return *(SDataRow *)SL_GET_NODE_DATA(node); return (SDataRow)SL_GET_NODE_DATA(node);
} }
static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
SDataRow row = tsdbNextIterRow(pIter); SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) return -1; if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
return dataRowKey(row); return dataRowKey(row);
} }
static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
SDataRow row = tsdbNextIterRow(pIter);
if (row == NULL) return TKEY_NULL;
return dataRowTKey(row);
}
static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
ASSERT(pRepo != NULL); ASSERT(pRepo != NULL);
if (pRepo->mem == NULL) return NULL; if (pRepo->mem == NULL) return NULL;

7
src/tsdb/src/tsdbMain.c
View File

@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
} }
} }
// update check
if (pCfg->update != 0) pCfg->update = 1;
return 0; return 0;
_err: _err:
@ -762,7 +765,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY
return -1; return -1;
} }
if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1; if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1;
(*affectedrows)++; (*affectedrows)++;
points++; points++;
@ -923,6 +926,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) {
tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock); tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock);
tlen += taosEncodeFixedI8(buf, pCfg->precision); tlen += taosEncodeFixedI8(buf, pCfg->precision);
tlen += taosEncodeFixedI8(buf, pCfg->compression); tlen += taosEncodeFixedI8(buf, pCfg->compression);
tlen += taosEncodeFixedI8(buf, pCfg->update);
return tlen; return tlen;
} }
@ -939,6 +943,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock)); buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock));
buf = taosDecodeFixedI8(buf, &(pCfg->precision)); buf = taosDecodeFixedI8(buf, &(pCfg->precision));
buf = taosDecodeFixedI8(buf, &(pCfg->compression)); buf = taosDecodeFixedI8(buf, &(pCfg->compression));
buf = taosDecodeFixedI8(buf, &(pCfg->update));
return buf; return buf;
} }

232
src/tsdb/src/tsdbMemTable.c
View File

@ -32,43 +32,40 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row);
// ---------------- INTERNAL FUNCTIONS ---------------- // ---------------- INTERNAL FUNCTIONS ----------------
int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
STsdbCfg * pCfg = &pRepo->config; STsdbCfg * pCfg = &pRepo->config;
STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbMeta * pMeta = pRepo->tsdbMeta;
int32_t level = 0; TKEY tkey = dataRowTKey(row);
int32_t headSize = 0;
TSKEY key = dataRowKey(row); TSKEY key = dataRowKey(row);
SMemTable * pMemTable = pRepo->mem; SMemTable * pMemTable = pRepo->mem;
STableData *pTableData = NULL; STableData *pTableData = NULL;
SSkipList * pSList = NULL; bool isRowDelete = TKEY_IS_DELETED(tkey);
if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && if (isRowDelete) {
pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { if (!pCfg->update) {
pTableData = pMemTable->tData[TABLE_TID(pTable)]; tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
pSList = pTableData->pData; terrno = TSDB_CODE_TDB_INVALID_ACTION;
return -1;
} }
tSkipListNewNodeInfo(pSList, &level, &headSize); if (key > TABLE_LASTKEY(pTable)) {
tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(SDataRow *)); REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
if (pNode == NULL) { return 0;
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; }
return -1;
} }
void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
if (pRow == NULL) { if (pRow == NULL) {
tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s", tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno)); REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
free(pNode);
return -1; return -1;
} }
pNode->level = level;
dataRowCpy(pRow, row); dataRowCpy(pRow, row);
*(SDataRow *)SL_GET_NODE_DATA(pNode) = pRow;
// Operations above may change pRepo->mem, retake those values // Operations above may change pRepo->mem, retake those values
ASSERT(pRepo->mem != NULL); ASSERT(pRepo->mem != NULL);
@ -77,7 +74,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
if (TABLE_TID(pTable) >= pMemTable->maxTables) { if (TABLE_TID(pTable) >= pMemTable->maxTables) {
if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) { if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
tsdbFreeBytes(pRepo, pRow, dataRowLen(row)); tsdbFreeBytes(pRepo, pRow, dataRowLen(row));
free(pNode);
return -1; return -1;
} }
} }
@ -97,7 +93,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
" to table %s while create new table data object since %s", " to table %s while create new table data object since %s",
REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno)); REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno));
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
free(pNode);
return -1; return -1;
} }
@ -106,24 +101,31 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
if (tSkipListPut(pTableData->pData, pNode) == NULL) { int64_t oldSize = SL_SIZE(pTableData->pData);
if (tSkipListPut(pTableData->pData, pRow) == NULL) {
tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
free(pNode); } else {
int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize;
if (isRowDelete) {
if (TABLE_LASTKEY(pTable) == key) {
// TODO: need to update table last key here (may from file)
}
} else { } else {
if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
}
if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
if (pMemTable->keyLast < key) pMemTable->keyLast = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key;
pMemTable->numOfRows++; pMemTable->numOfRows += deltaSize;
if (pTableData->keyFirst > key) pTableData->keyFirst = key; if (pTableData->keyFirst > key) pTableData->keyFirst = key;
if (pTableData->keyLast < key) pTableData->keyLast = key; if (pTableData->keyLast < key) pTableData->keyLast = key;
pTableData->numOfRows++; pTableData->numOfRows += deltaSize;
ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData));
} }
tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key); isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
key);
return 0; return 0;
} }
@ -295,63 +297,124 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
return 0; return 0;
} }
/**
* This is an important function to load data or try to load data from memory skiplist iterator.
*
* This function load memory data until:
* 1. iterator ends
* 2. data key exceeds maxKey
* 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
* 4. operations in pCols not exceeds its max capacity if pCols is given
*
* The function tries to procceed AS MUSH AS POSSIBLE.
*/
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
TSKEY *filterKeys, int nFilterKeys) { TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL);
if (pIter == NULL) return 0; if (pIter == NULL) return 0;
STSchema *pSchema = NULL; STSchema *pSchema = NULL;
int numOfRows = 0; TSKEY rowKey = 0;
TSKEY keyNext = 0; TSKEY fKey = 0;
bool isRowDel = false;
int filterIter = 0; int filterIter = 0;
SDataRow row = NULL;
if (nFilterKeys != 0) { // for filter purpose memset(pMergeInfo, 0, sizeof(*pMergeInfo));
ASSERT(filterKeys != NULL); pMergeInfo->keyFirst = INT64_MAX;
keyNext = tsdbNextIterKey(pIter); pMergeInfo->keyLast = INT64_MIN;
if (keyNext < 0 || keyNext > maxKey) return numOfRows; if (pCols) tdResetDataCols(pCols);
void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE);
filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY)));
}
do { row = tsdbNextIterRow(pIter);
SDataRow row = tsdbNextIterRow(pIter); if (row == NULL || dataRowKey(row) > maxKey) {
if (row == NULL) break; rowKey = INT64_MAX;
isRowDel = false;
keyNext = dataRowKey(row);
if (keyNext > maxKey) break;
bool keyFiltered = false;
if (nFilterKeys != 0) {
while (true) {
if (filterIter >= nFilterKeys) break;
if (keyNext == filterKeys[filterIter]) {
keyFiltered = true;
filterIter++;
break;
} else if (keyNext < filterKeys[filterIter]) {
break;
} else { } else {
rowKey = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
while (true) {
if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
if (fKey < rowKey) {
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
filterIter++; filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
}
} else if (fKey > rowKey) {
if (isRowDel) {
pMergeInfo->rowsDeleteFailed++;
} else {
if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
}
tSkipListIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || dataRowKey(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
} else {
if (isRowDel) {
ASSERT(!keepDup);
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsDeleteSucceed++;
pMergeInfo->nOperations++;
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else {
if (keepDup) {
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsUpdated++;
pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else {
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey);
}
}
tSkipListIterNext(pIter);
row = tsdbNextIterRow(pIter);
if (row == NULL || dataRowKey(row) > maxKey) {
rowKey = INT64_MAX;
isRowDel = false;
} else {
rowKey = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
filterIter++;
if (filterIter >= nFilterKeys) {
fKey = INT64_MAX;
} else {
fKey = tdGetKey(filterKeys[filterIter]);
} }
} }
} }
if (!keyFiltered) { return 0;
if (numOfRows >= maxRowsToRead) break;
if (pCols) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
if (pSchema == NULL) {
ASSERT(0);
}
}
tdAppendDataRowToDataCol(row, pSchema, pCols);
}
numOfRows++;
}
} while (tSkipListIterNext(pIter));
return numOfRows;
} }
// ---------------- LOCAL FUNCTIONS ---------------- // ---------------- LOCAL FUNCTIONS ----------------
@ -440,8 +503,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) {
pTableData->keyLast = 0; pTableData->keyLast = 0;
pTableData->numOfRows = 0; pTableData->numOfRows = 0;
pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, pTableData->pData =
TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 1, tsdbGetTsTupleKey); tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP],
tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey);
if (pTableData->pData == NULL) { if (pTableData->pData == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err; goto _err;
@ -461,7 +525,7 @@ static void tsdbFreeTableData(STableData *pTableData) {
} }
} }
static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(*(SDataRow *)data); } static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
static void *tsdbCommitData(void *arg) { static void *tsdbCommitData(void *arg) {
STsdbRepo * pRepo = (STsdbRepo *)arg; STsdbRepo * pRepo = (STsdbRepo *)arg;
@ -583,7 +647,7 @@ static void tsdbEndCommit(STsdbRepo *pRepo) {
static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
for (int i = 0; i < nIters; i++) { for (int i = 0; i < nIters; i++) {
TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
} }
return 0; return 0;
} }
@ -781,3 +845,19 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
return 0; return 0;
} }
static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) {
if (pCols) {
if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) {
*ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
if (*ppSchema == NULL) {
ASSERT(false);
return -1;
}
}
tdAppendDataRowToDataCol(row, *ppSchema, pCols);
}
return 0;
}

36
src/tsdb/src/tsdbMeta.c
View File

@ -86,7 +86,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
if (pTable != NULL) { if (pTable != NULL) {
tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
TABLE_TID(pTable), TABLE_UID(pTable)); TABLE_TID(pTable), TABLE_UID(pTable));
return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
goto _err;
} }
if (pCfg->type == TSDB_CHILD_TABLE) { if (pCfg->type == TSDB_CHILD_TABLE) {
@ -643,7 +644,7 @@ static void tsdbOrgMeta(void *pHandle) {
} }
static char *getTagIndexKey(const void *pData) { static char *getTagIndexKey(const void *pData) {
STable *pTable = *(STable **)pData; STable *pTable = (STable *)pData;
STSchema *pSchema = tsdbGetTableTagSchema(pTable); STSchema *pSchema = tsdbGetTableTagSchema(pTable);
STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN);
@ -700,7 +701,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
} }
pTable->tagVal = NULL; pTable->tagVal = NULL;
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) { if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err; goto _err;
@ -900,23 +901,8 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
pTable->pSuper = pSTable; pTable->pSuper = pSTable;
int32_t level = 0; tSkipListPut(pSTable->pIndex, (void *)pTable);
int32_t headSize = 0;
tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize);
// NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the
// actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function
SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *));
if (pNode == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return -1;
}
pNode->level = level;
memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *));
tSkipListPut(pSTable->pIndex, pNode);
if (refSuper) T_REF_INC(pSTable); if (refSuper) T_REF_INC(pSTable);
return 0; return 0;
} }
@ -940,7 +926,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
SSkipListNode *pNode = taosArrayGetP(res, i); SSkipListNode *pNode = taosArrayGetP(res, i);
// STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode);
if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need
tSkipListRemoveNode(pSTable->pIndex, pNode); tSkipListRemoveNode(pSTable->pIndex, pNode);
} }
} }
@ -1170,8 +1156,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
buf = tdDecodeSchema(buf, &(pTable->tagSchema)); buf = tdDecodeSchema(buf, &(pTable->tagSchema));
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
pTable->pIndex = pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL,
tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); SL_ALLOW_DUP_KEY, getTagIndexKey);
if (pTable->pIndex == NULL) { if (pTable->pIndex == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbFreeTable(pTable); tsdbFreeTable(pTable);
@ -1197,7 +1183,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) {
tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM); tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM);
} else { } else {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1)); tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1));
} else { } else {
tlen = sizeof(SListNode) + sizeof(SActObj); tlen = sizeof(SListNode) + sizeof(SActObj);
} }
@ -1244,7 +1230,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) {
} }
while (tSkipListIterNext(pIter)) { while (tSkipListIterNext(pIter)) {
STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE); ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE);
pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable); pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable);
} }
@ -1269,7 +1255,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) {
tsdbWLockRepoMeta(pRepo); tsdbWLockRepoMeta(pRepo);
while (tSkipListIterNext(pIter)) { while (tSkipListIterNext(pIter)) {
STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter));
tsdbRemoveTableFromMeta(pRepo, tTable, false, false); tsdbRemoveTableFromMeta(pRepo, tTable, false, false);
} }

300
src/tsdb/src/tsdbRWHelper.c
View File

@ -25,6 +25,7 @@
#define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
#define TSDB_KEY_COL_OFFSET 0 #define TSDB_KEY_COL_OFFSET 0
#define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock)) #define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock))
#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last)
static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock, static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock,
@ -32,7 +33,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD
static int compareKeyBlock(const void *arg1, const void *arg2); static int compareKeyBlock(const void *arg1, const void *arg2);
static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize); static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize);
static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo);
static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
static void tsdbResetHelperFileImpl(SRWHelper *pHelper); static void tsdbResetHelperFileImpl(SRWHelper *pHelper);
static int tsdbInitHelperFile(SRWHelper *pHelper); static int tsdbInitHelperFile(SRWHelper *pHelper);
@ -59,8 +60,10 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pComp
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock); static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock);
static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
int *blkIdx); int *blkIdx);
static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows); TSKEY maxKey, int maxRows, int8_t update);
static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps);
static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx);
// ---------------------- INTERNAL FUNCTIONS ---------------------- // ---------------------- INTERNAL FUNCTIONS ----------------------
int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
@ -277,7 +280,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols
while (true) { while (true) {
ASSERT(blkIdx <= (int)pIdx->numOfBlocks); ASSERT(blkIdx <= (int)pIdx->numOfBlocks);
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
if (keyFirst < 0 || keyFirst > maxKey) break; // iter over if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over
if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) { if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) {
if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1; if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1;
@ -923,7 +926,7 @@ _err:
return -1; return -1;
} }
static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) { static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) {
ASSERT(pCompBlock->numOfSubBlocks == 0); ASSERT(pCompBlock->numOfSubBlocks == 0);
SCompIdx *pIdx = &(pHelper->curCompIdx); SCompIdx *pIdx = &(pHelper->curCompIdx);
@ -956,9 +959,9 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
pSCompBlock->numOfSubBlocks++; pSCompBlock->numOfSubBlocks++;
ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
pSCompBlock->len += sizeof(SCompBlock); pSCompBlock->len += sizeof(SCompBlock);
pSCompBlock->numOfRows += rowsAdded; pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst); pSCompBlock->keyFirst = pMergeInfo->keyFirst;
pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast); pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += sizeof(SCompBlock); pIdx->len += sizeof(SCompBlock);
} else { // Need to create two sub-blocks } else { // Need to create two sub-blocks
void *ptr = NULL; void *ptr = NULL;
@ -987,11 +990,11 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
((SCompBlock *)ptr)[1] = *pCompBlock; ((SCompBlock *)ptr)[1] = *pCompBlock;
pSCompBlock->numOfSubBlocks = 2; pSCompBlock->numOfSubBlocks = 2;
pSCompBlock->numOfRows += rowsAdded; pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo); pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo);
pSCompBlock->len = sizeof(SCompBlock) * 2; pSCompBlock->len = sizeof(SCompBlock) * 2;
pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst); pSCompBlock->keyFirst = pMergeInfo->keyFirst;
pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast); pSCompBlock->keyLast = pMergeInfo->keyLast;
pIdx->len += (sizeof(SCompBlock) * 2); pIdx->len += (sizeof(SCompBlock) * 2);
} }
@ -1045,6 +1048,45 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
return 0; return 0;
} }
static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) {
SCompIdx *pCompIdx = &(pHelper->curCompIdx);
ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks);
SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx);
SCompBlock compBlock = *pCompBlock;
ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
if (pCompIdx->numOfBlocks == 1) {
memset(pCompIdx, 0, sizeof(*pCompIdx));
} else {
int tsize = 0;
if (compBlock.numOfSubBlocks > 1) {
tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks));
ASSERT(tsize > 0);
memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset),
POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks),
tsize);
pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks;
}
tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo));
ASSERT(tsize > 0);
memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize);
pCompIdx->len -= sizeof(SCompBlock);
pCompIdx->numOfBlocks--;
pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last);
pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast;
}
return 0;
}
static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
pHelper->idxH.numOfIdx = 0; pHelper->idxH.numOfIdx = 0;
pHelper->idxH.curIdx = 0; pHelper->idxH.curIdx = 0;
@ -1443,20 +1485,28 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
SCompBlock compBlock = {0}; SCompBlock compBlock = {0};
SMergeInfo mergeInfo = {0};
SMergeInfo *pMergeInfo = &mergeInfo;
ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey); ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey);
if (pIdx->hasLast) { // append to with last block if (pIdx->hasLast) { // append to with last block
ASSERT(pIdx->len > 0); ASSERT(pIdx->len > 0);
SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1);
ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock);
tdResetDataCols(pDataCols); tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols,
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, NULL, 0, pCfg->update, pMergeInfo);
pDataCols, NULL, 0);
ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
if (pDataCols->numOfRows > 0) {
ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1; pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast);
if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1;
} else { } else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows); ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
@ -1469,19 +1519,22 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
} }
if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
}
} else { } else {
ASSERT(!pHelper->hasOldLastBlock); ASSERT(!pHelper->hasOldLastBlock);
tdResetDataCols(pDataCols); tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows);
if (pDataCols->numOfRows > 0) {
ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
} }
}
#ifndef NDEBUG #ifndef NDEBUG
TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter); TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter);
ASSERT(keyNext < 0 || keyNext > pIdx->maxKey); ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey);
#endif #endif
return 0; return 0;
@ -1496,6 +1549,9 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
SDataCols * pDataCols0 = pHelper->pDataCols[0]; SDataCols * pDataCols0 = pHelper->pDataCols[0];
SMergeInfo mergeInfo = {0};
SMergeInfo *pMergeInfo = &mergeInfo;
SCompBlock oBlock = {0};
SSkipListIterator slIter = {0}; SSkipListIterator slIter = {0};
@ -1505,112 +1561,75 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE); pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE);
ASSERT(pCompBlock != NULL); ASSERT(pCompBlock != NULL);
int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock)); int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock));
oBlock = *pCompBlock;
if (pCompBlock->last) { ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1));
ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1);
int16_t colId = 0;
slIter = *(pCommitIter->pIter);
if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1;
ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows);
int rows1 = defaultRowsInBlock - pCompBlock->numOfRows; if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) {
int rows2 =
tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows);
if (rows2 == 0) { // all data filtered out
*(pCommitIter->pIter) = slIter;
} else {
if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock &&
pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols,
pDataCols0->cols[0].pData, pDataCols0->numOfRows);
ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows);
if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
tblkIdx++;
} else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
int round = 0;
int dIter = 0;
while (true) { while (true) {
tdResetDataCols(pDataCols); tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0,
int rowsRead = pCfg->update, pMergeInfo);
tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
if (rowsRead == 0) break; if (pDataCols->numOfRows == 0) break;
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
if (round == 0) {
if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
} else {
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
}
tblkIdx++;
round++;
}
}
if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
}
} else {
TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (pCompBlock[1].keyFirst - 1);
TSKEY blkKeyFirst = pCompBlock->keyFirst;
TSKEY blkKeyLast = pCompBlock->keyLast;
if (keyFirst < blkKeyFirst) {
while (true) {
tdResetDataCols(pDataCols);
int rowsRead =
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0);
if (rowsRead == 0) break;
ASSERT(rowsRead == pDataCols->numOfRows);
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
tblkIdx++; tblkIdx++;
} }
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
} else { } else {
ASSERT(keyFirst <= blkKeyLast);
int16_t colId = 0; int16_t colId = 0;
if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1;
TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1);
slIter = *(pCommitIter->pIter); slIter = *(pCommitIter->pIter);
int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows,
int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData, pCfg->update, pMergeInfo);
pDataCols0->numOfRows);
if (rows2 == 0) { // all filtered out if (pMergeInfo->nOperations == 0) {
// Do nothing
ASSERT(pMergeInfo->rowsDeleteFailed >= 0);
*(pCommitIter->pIter) = slIter; *(pCommitIter->pIter) = slIter;
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
} else {
int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2;
if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) {
int rows = (rows1 >= rows3) ? rows3 : rows2;
tdResetDataCols(pDataCols);
int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols,
pDataCols0->cols[0].pData, pDataCols0->numOfRows);
ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows);
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0)
return -1;
if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1;
tblkIdx++; tblkIdx++;
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) {
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); // Delete the block and do some stuff
ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1;
*pCommitIter->pIter = slIter;
if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
} else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) {
// Append as a sub-block of the searched block
tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData,
pDataCols0->numOfRows, pCfg->update, pMergeInfo);
ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0);
if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols,
&compBlock, oBlock.last, false) < 0) {
return -1;
}
if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) {
return -1;
}
tblkIdx++;
} else { } else {
if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; // load the block data, merge with the memory data
if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1;
int round = 0; int round = 0;
int dIter = 0; int dIter = 0;
while (true) { while (true) {
int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock,
tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock); pCfg->update);
if (rowsRead == 0) break;
if (pDataCols->numOfRows == 0) break;
if (tblkIdx == pIdx->numOfBlocks - 1) {
if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
} else {
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
}
if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0)
return -1;
if (round == 0) { if (round == 0) {
if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
} else { } else {
if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
@ -1619,10 +1638,6 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
round++; round++;
tblkIdx++; tblkIdx++;
} }
ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 ||
tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
}
}
} }
} }
@ -1630,9 +1645,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
return 0; return 0;
} }
static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
TSKEY maxKey, int maxRows) { TSKEY maxKey, int maxRows, int8_t update) {
int numOfRows = 0;
TSKEY key1 = INT64_MAX; TSKEY key1 = INT64_MAX;
TSKEY key2 = INT64_MAX; TSKEY key2 = INT64_MAX;
STSchema *pSchema = NULL; STSchema *pSchema = NULL;
@ -1642,35 +1656,62 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte
while (true) { while (true) {
key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
bool isRowDel = false;
SDataRow row = tsdbNextIterRow(pCommitIter->pIter); SDataRow row = tsdbNextIterRow(pCommitIter->pIter);
key2 = (row == NULL || dataRowKey(row) > maxKey) ? INT64_MAX : dataRowKey(row); if (row == NULL || dataRowKey(row) > maxKey) {
key2 = INT64_MAX;
} else {
key2 = dataRowKey(row);
isRowDel = dataRowDeleted(row);
}
if (key1 == INT64_MAX && key2 == INT64_MAX) break; if (key1 == INT64_MAX && key2 == INT64_MAX) break;
if (key1 <= key2) { if (key1 < key2) {
for (int i = 0; i < pDataCols->numOfCols; i++) { for (int i = 0; i < pDataCols->numOfCols; i++) {
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints); pTarget->maxPoints);
} }
pTarget->numOfRows++; pTarget->numOfRows++;
(*iter)++; (*iter)++;
if (key1 == key2) tSkipListIterNext(pCommitIter->pIter); } else if (key1 > key2) {
} else { if (!isRowDel) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
ASSERT(pSchema != NULL); ASSERT(pSchema != NULL);
} }
tdAppendDataRowToDataCol(row, pSchema, pTarget); tdAppendDataRowToDataCol(row, pSchema, pTarget);
}
tSkipListIterNext(pCommitIter->pIter);
} else {
if (update) {
if (!isRowDel) {
if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
ASSERT(pSchema != NULL);
}
tdAppendDataRowToDataCol(row, pSchema, pTarget);
}
} else {
ASSERT(!isRowDel);
for (int i = 0; i < pDataCols->numOfCols; i++) {
dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
pTarget->maxPoints);
}
pTarget->numOfRows++;
}
(*iter)++;
tSkipListIterNext(pCommitIter->pIter); tSkipListIterNext(pCommitIter->pIter);
} }
numOfRows++; if (pTarget->numOfRows >= maxRows) break;
if (numOfRows >= maxRows) break;
ASSERT(numOfRows == pTarget->numOfRows && numOfRows <= pTarget->maxPoints);
} }
return numOfRows;
} }
static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) { static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) {
@ -1693,3 +1734,20 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols,
return 0; return 0;
} }
static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) {
STsdbCfg *pCfg = &(pHelper->pRepo->config);
int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
ASSERT(mergeRows > 0);
if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) {
if (pCompBlock->last) {
if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true;
} else {
if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
}
}
return false;
}

57
src/tsdb/src/tsdbRead.c
View File

@ -457,7 +457,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
assert(node != NULL); assert(node != NULL);
SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@ -479,7 +479,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
assert(node != NULL); assert(node != NULL);
SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
@ -504,19 +504,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
tSkipListDestroyIter(pCheckInfo->iiter); tSkipListDestroyIter(pCheckInfo->iiter);
} }
static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
SDataRow rmem = NULL, rimem = NULL; SDataRow rmem = NULL, rimem = NULL;
if (pCheckInfo->iter) { if (pCheckInfo->iter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
if (node != NULL) { if (node != NULL) {
rmem = *(SDataRow *)SL_GET_NODE_DATA(node); rmem = (SDataRow)SL_GET_NODE_DATA(node);
} }
} }
if (pCheckInfo->iiter) { if (pCheckInfo->iiter) {
SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
if (node != NULL) { if (node != NULL) {
rimem = *(SDataRow *)SL_GET_NODE_DATA(node); rimem = (SDataRow)SL_GET_NODE_DATA(node);
} }
} }
@ -538,9 +538,15 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order
TSKEY r2 = dataRowKey(rimem); TSKEY r2 = dataRowKey(rimem);
if (r1 == r2) { // data ts are duplicated, ignore the data in mem if (r1 == r2) { // data ts are duplicated, ignore the data in mem
if (!update) {
tSkipListIterNext(pCheckInfo->iter); tSkipListIterNext(pCheckInfo->iter);
pCheckInfo->chosen = 1; pCheckInfo->chosen = 1;
return rimem; return rimem;
} else {
tSkipListIterNext(pCheckInfo->iiter);
pCheckInfo->chosen = 0;
return rmem;
}
} else { } else {
if (ASCENDING_TRAVERSE(order)) { if (ASCENDING_TRAVERSE(order)) {
if (r1 < r2) { if (r1 < r2) {
@ -594,6 +600,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
} }
static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
STsdbCfg *pCfg = &pHandle->pTsdb->config;
size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
pHandle->cur.fid = -1; pHandle->cur.fid = -1;
@ -607,7 +614,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
initTableMemIterator(pHandle, pCheckInfo); initTableMemIterator(pHandle, pCheckInfo);
} }
SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order); SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update);
if (row == NULL) { if (row == NULL) {
return false; return false;
} }
@ -827,11 +834,12 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pQueryHandle->cur;
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
/*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
assert(cur->pos >= 0 && cur->pos <= binfo.rows); assert(cur->pos >= 0 && cur->pos <= binfo.rows);
@ -1316,6 +1324,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
SQueryFilePos* cur = &pQueryHandle->cur; SQueryFilePos* cur = &pQueryHandle->cur;
SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
initTableMemIterator(pQueryHandle, pCheckInfo); initTableMemIterator(pQueryHandle, pCheckInfo);
@ -1353,7 +1362,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
} else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
SSkipListNode* node = NULL; SSkipListNode* node = NULL;
do { do {
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) { if (row == NULL) {
break; break;
} }
@ -1383,7 +1392,22 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
moveToNextRowInMem(pCheckInfo); moveToNextRowInMem(pCheckInfo);
} else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it
if (pCfg->update) {
copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable);
numOfRows += 1;
if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = key;
}
cur->win.ekey = key;
cur->lastKey = key + step;
cur->mixBlock = true;
moveToNextRowInMem(pCheckInfo); moveToNextRowInMem(pCheckInfo);
pos += step;
} else {
moveToNextRowInMem(pCheckInfo);
}
} else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
if (cur->win.skey == TSKEY_INITIAL_VAL) { if (cur->win.skey == TSKEY_INITIAL_VAL) {
@ -1394,7 +1418,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
assert(end != -1); assert(end != -1);
if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
if (!pCfg->update) {
moveToNextRowInMem(pCheckInfo); moveToNextRowInMem(pCheckInfo);
} else {
end -= step;
}
} }
int32_t qstart = 0, qend = 0; int32_t qstart = 0, qend = 0;
@ -1414,8 +1442,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
* copy them all to result buffer, since it may be overlapped with file data block. * copy them all to result buffer, since it may be overlapped with file data block.
*/ */
if (node == NULL || if (node == NULL ||
((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
// no data in cache or data in cache is greater than the ekey of time window, load data from file block // no data in cache or data in cache is greater than the ekey of time window, load data from file block
if (cur->win.skey == TSKEY_INITIAL_VAL) { if (cur->win.skey == TSKEY_INITIAL_VAL) {
cur->win.skey = tsArray[pos]; cur->win.skey = tsArray[pos];
@ -1862,13 +1890,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
STsdbQueryHandle* pQueryHandle) { STsdbQueryHandle* pQueryHandle) {
int numOfRows = 0; int numOfRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns);
STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
win->skey = TSKEY_INITIAL_VAL; win->skey = TSKEY_INITIAL_VAL;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
STable* pTable = pCheckInfo->pTableObj; STable* pTable = pCheckInfo->pTableObj;
do { do {
SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
if (row == NULL) { if (row == NULL) {
break; break;
} }
@ -1919,9 +1948,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
while (tSkipListIterNext(iter)) { while (tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter); SSkipListNode* pNode = tSkipListIterGet(iter);
STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode);
STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(list, &info); taosArrayPush(list, &info);
} }
@ -2464,7 +2493,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
static bool indexedNodeFilterFp(const void* pNode, void* param) { static bool indexedNodeFilterFp(const void* pNode, void* param) {
tQueryInfo* pInfo = (tQueryInfo*) param; tQueryInfo* pInfo = (tQueryInfo*) param;
STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
char* val = NULL; char* val = NULL;

2
src/util/CMakeLists.txt
View File

@ -8,7 +8,7 @@ TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z)
IF (TD_LINUX) IF (TD_LINUX)
TARGET_LINK_LIBRARIES(tutil m rt) TARGET_LINK_LIBRARIES(tutil m rt)
ADD_SUBDIRECTORY(tests) # ADD_SUBDIRECTORY(tests)
FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
IF (ICONV_INCLUDE_EXIST) IF (ICONV_INCLUDE_EXIST)

42
src/util/inc/tfile.h Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TFILE_H
#define TDENGINE_TFILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <unistd.h>
// init taos file module
int32_t tfinit();
// clean up taos file module
void tfcleanup();
// the same syntax as UNIX standard open/close/read/write
// but FD is int64_t and will never be reused
int64_t tfopen(const char *pathname, int32_t flags);
int64_t tfclose(int64_t tfd);
int64_t tfwrite(int64_t tfd, void *buf, int64_t count);
int64_t tfread(int64_t tfd, void *buf, int64_t count);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TREF_H

40
src/util/inc/tref.h
View File

@ -21,38 +21,48 @@
extern "C" { extern "C" {
#endif #endif
// open an instance, return refId which will be used by other APIs // open a reference set, max is the mod used by hash, fp is the pointer to free resource function
// return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately
int taosOpenRef(int max, void (*fp)(void *)); int taosOpenRef(int max, void (*fp)(void *));
// close the Ref instance // close the reference set, refId is the return value by taosOpenRef
void taosCloseRef(int refId); // return 0 if success. On error, -1 is returned, and terrno is set appropriately
int taosCloseRef(int refId);
// add ref, p is the pointer to resource or pointer ID // add ref, p is the pointer to resource or pointer ID
int taosAddRef(int refId, void *p); // return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately
#define taosRemoveRef taosReleaseRef int64_t taosAddRef(int refId, void *p);
// acquire ref, p is the pointer to resource or pointer ID // remove ref, rid is the reference ID returned by taosAddRef
int taosAcquireRef(int refId, void *p); // return 0 if success. On error, -1 is returned, and terrno is set appropriately
int taosRemoveRef(int rsetId, int64_t rid);
// release ref, p is the pointer to resource or pinter ID // acquire ref, rid is the reference ID returned by taosAddRef
void taosReleaseRef(int refId, void *p); // return the resource p. On error, NULL is returned, and terrno is set appropriately
void *taosAcquireRef(int rsetId, int64_t rid);
// return the first if p is null, otherwise return the next after p // release ref, rid is the reference ID returned by taosAddRef
void *taosIterateRef(int refId, void *p); // return 0 if success. On error, -1 is returned, and terrno is set appropriately
int taosReleaseRef(int rsetId, int64_t rid);
// return the first reference if rid is 0, otherwise return the next after current reference.
// if return value is NULL, it means list is over(if terrno is set, it means error happens)
void *taosIterateRef(int rsetId, int64_t rid);
// return the number of references in system // return the number of references in system
int taosListRef(); int taosListRef();
/* sample code to iterate the refs /* sample code to iterate the refs
void demoIterateRefs(int refId) { void demoIterateRefs(int rsetId) {
void *p = taosIterateRef(refId, NULL); void *p = taosIterateRef(refId, 0);
while (p) { while (p) {
// process P // process P
p = taosIterateRef(refId, p); // get the rid from p
p = taosIterateRef(rsetId, rid);
} }
} }

163
src/util/inc/tskiplist.h
View File

@ -27,33 +27,25 @@ extern "C" {
#define MAX_SKIP_LIST_LEVEL 15 #define MAX_SKIP_LIST_LEVEL 15
#define SKIP_LIST_RECORD_PERFORMANCE 0 #define SKIP_LIST_RECORD_PERFORMANCE 0
// For key property setting
#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage)
#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case)
#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update=1 case)
// For thread safety setting
#define SL_THREAD_SAFE (uint8_t)0x4
typedef char *SSkipListKey; typedef char *SSkipListKey;
typedef char *(*__sl_key_fn_t)(const void *); typedef char *(*__sl_key_fn_t)(const void *);
/**
* the skip list node is located in a consecutive memory area,
* the format of skip list node is as follows:
* +------------+-----------------------+------------------------+-----+------+
* | node level | forward pointer array | backward pointer array | key | data |
* +------------+-----------------------+------------------------+-----+------+
*/
typedef struct SSkipListNode { typedef struct SSkipListNode {
uint8_t level; uint8_t level;
void * pData;
struct SSkipListNode *forwards[];
} SSkipListNode; } SSkipListNode;
#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES) #define SL_GET_NODE_DATA(n) (n)->pData
#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)]
#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)] #define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)]
#define SL_GET_BACKWARD_POINTER(n, _l) \
((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)]
#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level))
#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n)))
#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0)))
#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0)))
#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n))
/* /*
* @version 0.3 * @version 0.3
@ -103,21 +95,16 @@ typedef struct tSkipListState {
uint64_t nTotalElapsedTimeForInsert; uint64_t nTotalElapsedTimeForInsert;
} tSkipListState; } tSkipListState;
typedef struct SSkipListKeyInfo {
uint8_t dupKey : 2; // if allow duplicated key in the skip list
uint8_t type : 4; // key type
uint8_t freeNode:2; // free node when destroy the skiplist
uint8_t len; // maximum key length, used in case of string key
} SSkipListKeyInfo;
typedef struct SSkipList { typedef struct SSkipList {
__compar_fn_t comparFn; __compar_fn_t comparFn;
__sl_key_fn_t keyFn; __sl_key_fn_t keyFn;
uint32_t size;
uint8_t maxLevel;
uint8_t level;
SSkipListKeyInfo keyInfo;
pthread_rwlock_t *lock; pthread_rwlock_t *lock;
uint16_t len;
uint8_t maxLevel;
uint8_t flags;
uint8_t type; // static info above
uint8_t level;
uint32_t size;
SSkipListNode * pHead; // point to the first element SSkipListNode * pHead; // point to the first element
SSkipListNode * pTail; // point to the last element SSkipListNode * pTail; // point to the last element
#if SKIP_LIST_RECORD_PERFORMANCE #if SKIP_LIST_RECORD_PERFORMANCE
@ -125,12 +112,6 @@ typedef struct SSkipList {
#endif #endif
} SSkipList; } SSkipList;
/*
* iterate the skiplist
* this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may
* continue iterating the skiplist, so add the reference count for skiplist
* TODO add the ref for skip list when one iterator is created
*/
typedef struct SSkipListIterator { typedef struct SSkipListIterator {
SSkipList * pSkipList; SSkipList * pSkipList;
SSkipListNode *cur; SSkipListNode *cur;
@ -139,113 +120,25 @@ typedef struct SSkipListIterator {
SSkipListNode *next; // next points to the true qualified node in skip list SSkipListNode *next; // next points to the true qualified node in skip list
} SSkipListIterator; } SSkipListIterator;
/** #define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE)
* #define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1))
* @param nMaxLevel maximum skip list level #define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData))
* @param keyType type of key #define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0))
* @param dupKey allow the duplicated key in the skip list #define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0))
* @return #define SL_SIZE(s) (s)->size
*/
SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe,
uint8_t freeNode, __sl_key_fn_t fn);
/** SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags,
* __sl_key_fn_t fn);
* @param pSkipList void tSkipListDestroy(SSkipList *pSkipList);
* @return NULL will always be returned SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData);
*/
void *tSkipListDestroy(SSkipList *pSkipList);
/**
*
* @param pSkipList
* @param level
* @param headSize
*/
void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize);
/**
* put the skip list node into the skip list.
* If failed, NULL will be returned, otherwise, the pNode will be returned.
*
* @param pSkipList
* @param pNode
* @return
*/
SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode);
/**
* get *all* nodes which key are equivalent to pKey
*
* @param pSkipList
* @param pKey
* @return
*/
SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
/**
* get the size of skip list
* @param pSkipList
* @return
*/
size_t tSkipListGetSize(const SSkipList *pSkipList);
/**
* display skip list of the given level, for debug purpose only
* @param pSkipList
* @param nlevel
*/
void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
/**
* create skiplist iterator
* @param pSkipList
* @return
*/
SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);
/**
* create skip list iterator from the given node and specified the order
* @param pSkipList
* @param pNode start position, instead of the first node in skip list
* @param order traverse order of the iterator
* @return
*/
SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order);
/**
* forward the skip list iterator
* @param iter
* @return
*/
bool tSkipListIterNext(SSkipListIterator *iter); bool tSkipListIterNext(SSkipListIterator *iter);
/**
* get the element of skip list node
* @param iter
* @return
*/
SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); SSkipListNode * tSkipListIterGet(SSkipListIterator *iter);
/**
* destroy the skip list node
* @param iter
* @return
*/
void * tSkipListDestroyIter(SSkipListIterator *iter); void * tSkipListDestroyIter(SSkipListIterator *iter);
/*
* remove nodes of the pKey value.
* If more than one node has the same value, all will be removed
*
* @Return
* the count of removed nodes
*/
uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key);
/*
* remove the specified node in parameters
*/
void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode);
#ifdef __cplusplus #ifdef __cplusplus

81
src/util/src/tfile.c Normal file
View File

@ -0,0 +1,81 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taoserror.h"
#include "tulog.h"
#include "tutil.h"
#include "tref.h"
static int32_t tsFileRsetId = -1;
static void taosCloseFile(void *p) {
close((int32_t)(uintptr_t)p);
}
int32_t tfinit() {
tsFileRsetId = taosOpenRef(2000, taosCloseFile);
return tsFileRsetId;
}
void tfcleanup() {
if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
tsFileRsetId = -1;
}
int64_t tfopen(const char *pathname, int32_t flags) {
int32_t fd = open(pathname, flags);
if (fd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
void *p = (void *)(int64_t)fd;
int64_t rid = taosAddRef(tsFileRsetId, p);
if (rid < 0) close(fd);
return rid;
}
int64_t tfclose(int64_t tfd) {
return taosRemoveRef(tsFileRsetId, tfd);
}
int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int64_t ret = taosWrite(fd, buf, count);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}
int64_t tfread(int64_t tfd, void *buf, int64_t count) {
void *p = taosAcquireRef(tsFileRsetId, tfd);
if (p == NULL) return -1;
int32_t fd = (int32_t)(uintptr_t)p;
int64_t ret = taosRead(fd, buf, count);
if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseRef(tsFileRsetId, tfd);
return ret;
}

376
src/util/src/tref.c
View File

@ -24,17 +24,20 @@
#define TSDB_REF_STATE_DELETED 2 #define TSDB_REF_STATE_DELETED 2
typedef struct SRefNode { typedef struct SRefNode {
struct SRefNode *prev; struct SRefNode *prev; // previous node
struct SRefNode *next; struct SRefNode *next; // next node
void *p; void *p; // pointer to resource protected,
int32_t count; int64_t rid; // reference ID
int32_t count; // number of references
int removed; // 1: removed
} SRefNode; } SRefNode;
typedef struct { typedef struct {
SRefNode **nodeList; SRefNode **nodeList; // array of SRefNode linked list
int state; // 0: empty, 1: active; 2: deleted int state; // 0: empty, 1: active; 2: deleted
int refId; int rsetId; // refSet ID, global unique
int max; int64_t rid; // increase by one for each new reference
int max; // mod
int32_t count; // total number of SRefNodes in this set int32_t count; // total number of SRefNodes in this set
int64_t *lockedBy; int64_t *lockedBy;
void (*fp)(void *); void (*fp)(void *);
@ -47,54 +50,58 @@ static int tsRefSetNum = 0;
static int tsNextId = 0; static int tsNextId = 0;
static void taosInitRefModule(void); static void taosInitRefModule(void);
static int taosHashRef(SRefSet *pSet, void *p);
static void taosLockList(int64_t *lockedBy); static void taosLockList(int64_t *lockedBy);
static void taosUnlockList(int64_t *lockedBy); static void taosUnlockList(int64_t *lockedBy);
static void taosIncRefCount(SRefSet *pSet); static void taosIncRsetCount(SRefSet *pSet);
static void taosDecRefCount(SRefSet *pSet); static void taosDecRsetCount(SRefSet *pSet);
static int taosDecRefCount(int rsetId, int64_t rid, int remove);
int taosOpenRef(int max, void (*fp)(void *)) int taosOpenRef(int max, void (*fp)(void *))
{ {
SRefNode **nodeList; SRefNode **nodeList;
SRefSet *pSet; SRefSet *pSet;
int64_t *lockedBy; int64_t *lockedBy;
int i, refId; int i, rsetId;
pthread_once(&tsRefModuleInit, taosInitRefModule); pthread_once(&tsRefModuleInit, taosInitRefModule);
nodeList = calloc(sizeof(SRefNode *), (size_t)max); nodeList = calloc(sizeof(SRefNode *), (size_t)max);
if (nodeList == NULL) { if (nodeList == NULL) {
return TSDB_CODE_REF_NO_MEMORY; terrno = TSDB_CODE_REF_NO_MEMORY;
return -1;
} }
lockedBy = calloc(sizeof(int64_t), (size_t)max); lockedBy = calloc(sizeof(int64_t), (size_t)max);
if (lockedBy == NULL) { if (lockedBy == NULL) {
free(nodeList); free(nodeList);
return TSDB_CODE_REF_NO_MEMORY; terrno = TSDB_CODE_REF_NO_MEMORY;
return -1;
} }
pthread_mutex_lock(&tsRefMutex); pthread_mutex_lock(&tsRefMutex);
for (i = 0; i < TSDB_REF_OBJECTS; ++i) { for (i = 0; i < TSDB_REF_OBJECTS; ++i) {
tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS;
if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId
if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break; if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break;
} }
if (i < TSDB_REF_OBJECTS) { if (i < TSDB_REF_OBJECTS) {
refId = tsNextId; rsetId = tsNextId;
pSet = tsRefSetList + refId; pSet = tsRefSetList + rsetId;
taosIncRefCount(pSet);
pSet->max = max; pSet->max = max;
pSet->nodeList = nodeList; pSet->nodeList = nodeList;
pSet->lockedBy = lockedBy; pSet->lockedBy = lockedBy;
pSet->fp = fp; pSet->fp = fp;
pSet->rid = 1;
pSet->rsetId = rsetId;
pSet->state = TSDB_REF_STATE_ACTIVE; pSet->state = TSDB_REF_STATE_ACTIVE;
pSet->refId = refId; taosIncRsetCount(pSet);
tsRefSetNum++; tsRefSetNum++;
uTrace("refId:%d is opened, max:%d, fp:%p refSetNum:%d", refId, max, fp, tsRefSetNum); uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum);
} else { } else {
refId = TSDB_CODE_REF_FULL; rsetId = TSDB_CODE_REF_FULL;
free (nodeList); free (nodeList);
free (lockedBy); free (lockedBy);
uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum);
@ -102,239 +109,205 @@ int taosOpenRef(int max, void (*fp)(void *))
pthread_mutex_unlock(&tsRefMutex); pthread_mutex_unlock(&tsRefMutex);
return refId; return rsetId;
} }
void taosCloseRef(int refId) int taosCloseRef(int rsetId)
{ {
SRefSet *pSet; SRefSet *pSet;
int deleted = 0; int deleted = 0;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) { if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d is invalid, out of range", refId); uTrace("rsetId:%d is invalid, out of range", rsetId);
return; terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
} }
pSet = tsRefSetList + refId; pSet = tsRefSetList + rsetId;
pthread_mutex_lock(&tsRefMutex); pthread_mutex_lock(&tsRefMutex);
if (pSet->state == TSDB_REF_STATE_ACTIVE) { if (pSet->state == TSDB_REF_STATE_ACTIVE) {
pSet->state = TSDB_REF_STATE_DELETED; pSet->state = TSDB_REF_STATE_DELETED;
deleted = 1; deleted = 1;
uTrace("refId:%d is closed, count:%d", refId, pSet->count); uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count);
} else { } else {
uTrace("refId:%d is already closed, count:%d", refId, pSet->count); uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count);
} }
pthread_mutex_unlock(&tsRefMutex); pthread_mutex_unlock(&tsRefMutex);
if (deleted) taosDecRefCount(pSet); if (deleted) taosDecRsetCount(pSet);
return 0;
} }
int taosAddRef(int refId, void *p) int64_t taosAddRef(int rsetId, void *p)
{ {
int hash; int hash;
SRefNode *pNode; SRefNode *pNode;
SRefSet *pSet; SRefSet *pSet;
int64_t rid = 0;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) { if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d p:%p failed to add, refId not valid", refId, p); uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p);
return TSDB_CODE_REF_INVALID_ID; terrno = TSDB_CODE_REF_INVALID_ID;
return -1;
} }
pSet = tsRefSetList + refId; pSet = tsRefSetList + rsetId;
taosIncRefCount(pSet); taosIncRsetCount(pSet);
if (pSet->state != TSDB_REF_STATE_ACTIVE) { if (pSet->state != TSDB_REF_STATE_ACTIVE) {
taosDecRefCount(pSet); taosDecRsetCount(pSet);
uTrace("refId:%d p:%p failed to add, not active", refId, p); uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p);
return TSDB_CODE_REF_ID_REMOVED; terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
} }
int code = 0; pNode = calloc(sizeof(SRefNode), 1);
hash = taosHashRef(pSet, p); if (pNode == NULL) {
terrno = TSDB_CODE_REF_NO_MEMORY;
return -1;
}
rid = atomic_add_fetch_64(&pSet->rid, 1);
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash); taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->p == p)
break;
pNode = pNode->next;
}
if (pNode) {
code = TSDB_CODE_REF_ALREADY_EXIST;
uTrace("refId:%d p:%p is already there, faild to add", refId, p);
} else {
pNode = calloc(sizeof(SRefNode), 1);
if (pNode) {
pNode->p = p; pNode->p = p;
pNode->rid = rid;
pNode->count = 1; pNode->count = 1;
pNode->prev = 0;
pNode->prev = NULL;
pNode->next = pSet->nodeList[hash]; pNode->next = pSet->nodeList[hash];
if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode; if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode;
pSet->nodeList[hash] = pNode; pSet->nodeList[hash] = pNode;
uTrace("refId:%d p:%p is added, count:%d malloc mem: %p", refId, p, pSet->count, pNode);
} else {
code = TSDB_CODE_REF_NO_MEMORY;
uTrace("refId:%d p:%p is not added, since no memory", refId, p);
}
}
if (code < 0) taosDecRefCount(pSet); uTrace("rsetId:%d p:%p rid:%" PRId64 " is added, count:%d", rsetId, p, rid, pSet->count);
taosUnlockList(pSet->lockedBy+hash); taosUnlockList(pSet->lockedBy+hash);
return code; return rid;
} }
int taosAcquireRef(int refId, void *p) int taosRemoveRef(int rsetId, int64_t rid)
{ {
int hash, code = 0; return taosDecRefCount(rsetId, rid, 1);
SRefNode *pNode;
SRefSet *pSet;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d p:%p failed to acquire, refId not valid", refId, p);
return TSDB_CODE_REF_INVALID_ID;
} }
pSet = tsRefSetList + refId; // if rid is 0, return the first p in hash list, otherwise, return the next after current rid
taosIncRefCount(pSet); void *taosAcquireRef(int rsetId, int64_t rid)
if (pSet->state != TSDB_REF_STATE_ACTIVE) {
uTrace("refId:%d p:%p failed to acquire, not active", refId, p);
taosDecRefCount(pSet);
return TSDB_CODE_REF_ID_REMOVED;
}
hash = taosHashRef(pSet, p);
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->p == p) {
break;
}
pNode = pNode->next;
}
if (pNode) {
pNode->count++;
uTrace("refId:%d p:%p is acquired", refId, p);
} else {
code = TSDB_CODE_REF_NOT_EXIST;
uTrace("refId:%d p:%p is not there, failed to acquire", refId, p);
}
taosUnlockList(pSet->lockedBy+hash);
taosDecRefCount(pSet);
return code;
}
void taosReleaseRef(int refId, void *p)
{ {
int hash; int hash;
SRefNode *pNode; SRefNode *pNode;
SRefSet *pSet; SRefSet *pSet;
int released = 0; void *p = NULL;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) { if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d p:%p failed to release, refId not valid", refId, p); uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid);
return; terrno = TSDB_CODE_REF_INVALID_ID;
return NULL;
} }
pSet = tsRefSetList + refId; if (rid <= 0) {
if (pSet->state == TSDB_REF_STATE_EMPTY) { uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rid not valid", rsetId, rid);
uTrace("refId:%d p:%p failed to release, cleaned", refId, p); terrno = TSDB_CODE_REF_NOT_EXIST;
return; return NULL;
} }
hash = taosHashRef(pSet, p); pSet = tsRefSetList + rsetId;
taosIncRsetCount(pSet);
if (pSet->state != TSDB_REF_STATE_ACTIVE) {
uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, not active", rsetId, rid);
taosDecRsetCount(pSet);
terrno = TSDB_CODE_REF_ID_REMOVED;
return NULL;
}
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash); taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash]; pNode = pSet->nodeList[hash];
while (pNode) { while (pNode) {
if (pNode->p == p) if (pNode->rid == rid) {
break; break;
}
pNode = pNode->next; pNode = pNode->next;
} }
if (pNode) { if (pNode) {
pNode->count--; if (pNode->removed == 0) {
pNode->count++;
if (pNode->count == 0) { p = pNode->p;
if ( pNode->prev ) { uTrace("rsetId:%d p:%p rid:%" PRId64 " is acquired", rsetId, pNode->p, rid);
pNode->prev->next = pNode->next;
} else { } else {
pSet->nodeList[hash] = pNode->next; terrno = TSDB_CODE_REF_NOT_EXIST;
} uTrace("rsetId:%d p:%p rid:%" PRId64 " is already removed, failed to acquire", rsetId, pNode->p, rid);
if ( pNode->next ) {
pNode->next->prev = pNode->prev;
}
(*pSet->fp)(pNode->p);
free(pNode);
released = 1;
uTrace("refId:%d p:%p is removed, count:%d, free mem: %p", refId, p, pSet->count, pNode);
} else {
uTrace("refId:%d p:%p is released", refId, p);
} }
} else { } else {
uTrace("refId:%d p:%p is not there, failed to release", refId, p); terrno = TSDB_CODE_REF_NOT_EXIST;
uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid);
} }
taosUnlockList(pSet->lockedBy+hash); taosUnlockList(pSet->lockedBy+hash);
if (released) taosDecRefCount(pSet); taosDecRsetCount(pSet);
return p;
} }
// if p is NULL, return the first p in hash list, otherwise, return the next after p int taosReleaseRef(int rsetId, int64_t rid)
void *taosIterateRef(int refId, void *p) { {
return taosDecRefCount(rsetId, rid, 0);
}
// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
void *taosIterateRef(int rsetId, int64_t rid) {
SRefNode *pNode = NULL; SRefNode *pNode = NULL;
SRefSet *pSet; SRefSet *pSet;
if (refId < 0 || refId >= TSDB_REF_OBJECTS) { if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
uTrace("refId:%d p:%p failed to iterate, refId not valid", refId, p); uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rsetId not valid", rsetId, rid);
terrno = TSDB_CODE_REF_INVALID_ID;
return NULL; return NULL;
} }
pSet = tsRefSetList + refId; if (rid <= 0) {
taosIncRefCount(pSet); uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rid not valid", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
return NULL;
}
pSet = tsRefSetList + rsetId;
taosIncRsetCount(pSet);
if (pSet->state != TSDB_REF_STATE_ACTIVE) { if (pSet->state != TSDB_REF_STATE_ACTIVE) {
uTrace("refId:%d p:%p failed to iterate, not active", refId, p); uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rset not active", rsetId, rid);
taosDecRefCount(pSet); terrno = TSDB_CODE_REF_ID_REMOVED;
taosDecRsetCount(pSet);
return NULL; return NULL;
} }
int hash = 0; int hash = 0;
if (p) { if (rid > 0) {
hash = taosHashRef(pSet, p); hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash); taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash]; pNode = pSet->nodeList[hash];
while (pNode) { while (pNode) {
if (pNode->p == p) break; if (pNode->rid == rid) break;
pNode = pNode->next; pNode = pNode->next;
} }
if (pNode == NULL) { if (pNode == NULL) {
uError("refId:%d p:%p not there, quit", refId, p); uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
taosUnlockList(pSet->lockedBy+hash); taosUnlockList(pSet->lockedBy+hash);
return NULL; return NULL;
} }
// p is there // rid is there
pNode = pNode->next; pNode = pNode->next;
if (pNode == NULL) { if (pNode == NULL) {
taosUnlockList(pSet->lockedBy+hash); taosUnlockList(pSet->lockedBy+hash);
@ -356,14 +329,14 @@ void *taosIterateRef(int refId, void *p) {
pNode->count++; // acquire it pNode->count++; // acquire it
newP = pNode->p; newP = pNode->p;
taosUnlockList(pSet->lockedBy+hash); taosUnlockList(pSet->lockedBy+hash);
uTrace("refId:%d p:%p is returned", refId, p); uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid);
} else { } else {
uTrace("refId:%d p:%p the list is over", refId, p); uTrace("rsetId:%d the list is over", rsetId);
} }
if (p) taosReleaseRef(refId, p); // release the current one if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one
taosDecRefCount(pSet); taosDecRsetCount(pSet);
return newP; return newP;
} }
@ -381,13 +354,13 @@ int taosListRef() {
if (pSet->state == TSDB_REF_STATE_EMPTY) if (pSet->state == TSDB_REF_STATE_EMPTY)
continue; continue;
uInfo("refId:%d state:%d count::%d", i, pSet->state, pSet->count); uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count);
for (int j=0; j < pSet->max; ++j) { for (int j=0; j < pSet->max; ++j) {
pNode = pSet->nodeList[j]; pNode = pSet->nodeList[j];
while (pNode) { while (pNode) {
uInfo("refId:%d p:%p count:%d", i, pNode->p, pNode->count); uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count);
pNode = pNode->next; pNode = pNode->next;
num++; num++;
} }
@ -399,20 +372,77 @@ int taosListRef() {
return num; return num;
} }
static int taosHashRef(SRefSet *pSet, void *p) static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
{ int hash;
int hash = 0; SRefSet *pSet;
int64_t v = (int64_t)p; SRefNode *pNode;
int released = 0;
int code = 0;
for (int i = 0; i < sizeof(v); ++i) { if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
hash += (int)(v & 0xFFFF); uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid);
v = v >> 16; terrno = TSDB_CODE_REF_INVALID_ID;
i = i + 2; return -1;
} }
hash = hash % pSet->max; if (rid <= 0) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
return -1;
}
return hash; pSet = tsRefSetList + rsetId;
if (pSet->state == TSDB_REF_STATE_EMPTY) {
uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid);
terrno = TSDB_CODE_REF_ID_REMOVED;
return -1;
}
hash = rid % pSet->max;
taosLockList(pSet->lockedBy+hash);
pNode = pSet->nodeList[hash];
while (pNode) {
if (pNode->rid == rid)
break;
pNode = pNode->next;
}
if (pNode) {
pNode->count--;
if (remove) pNode->removed = 1;
if (pNode->count <= 0) {
if (pNode->prev) {
pNode->prev->next = pNode->next;
} else {
pSet->nodeList[hash] = pNode->next;
}
if (pNode->next) {
pNode->next->prev = pNode->prev;
}
(*pSet->fp)(pNode->p);
uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
free(pNode);
released = 1;
} else {
uTrace("rsetId:%d p:%p rid:%" PRId64 "is released, count:%d", rsetId, pNode->p, rid, pNode->count);
}
} else {
uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);
terrno = TSDB_CODE_REF_NOT_EXIST;
code = -1;
}
taosUnlockList(pSet->lockedBy+hash);
if (released) taosDecRsetCount(pSet);
return code;
} }
static void taosLockList(int64_t *lockedBy) { static void taosLockList(int64_t *lockedBy) {
@ -436,14 +466,14 @@ static void taosInitRefModule(void) {
pthread_mutex_init(&tsRefMutex, NULL); pthread_mutex_init(&tsRefMutex, NULL);
} }
static void taosIncRefCount(SRefSet *pSet) { static void taosIncRsetCount(SRefSet *pSet) {
atomic_add_fetch_32(&pSet->count, 1); atomic_add_fetch_32(&pSet->count, 1);
uTrace("refId:%d inc count:%d", pSet->refId, pSet->count); // uTrace("rsetId:%d inc count:%d", pSet->rsetId, count);
} }
static void taosDecRefCount(SRefSet *pSet) { static void taosDecRsetCount(SRefSet *pSet) {
int32_t count = atomic_sub_fetch_32(&pSet->count, 1); int32_t count = atomic_sub_fetch_32(&pSet->count, 1);
uTrace("refId:%d dec count:%d", pSet->refId, pSet->count); // uTrace("rsetId:%d dec count:%d", pSet->rsetId, count);
if (count > 0) return; if (count > 0) return;
@ -458,7 +488,7 @@ static void taosDecRefCount(SRefSet *pSet) {
taosTFree(pSet->lockedBy); taosTFree(pSet->lockedBy);
tsRefSetNum--; tsRefSetNum--;
uTrace("refId:%d is cleaned, refSetNum:%d count:%d", pSet->refId, tsRefSetNum, pSet->count); uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count);
} }
pthread_mutex_unlock(&tsRefMutex); pthread_mutex_unlock(&tsRefMutex);

960
src/util/src/tskiplist.c
View File

File diff suppressed because it is too large Load Diff

4
src/util/tests/skiplistTest.cpp
View File

@ -247,7 +247,7 @@ void skiplistPerformanceTest() {
printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size); printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size);
printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level); printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level);
assert(tSkipListGetSize(pSkipList) == size); assert(SL_GET_SIZE(pSkipList) == size);
// printf("the level of skiplist is:\n"); // printf("the level of skiplist is:\n");
// //
@ -273,7 +273,7 @@ void skiplistPerformanceTest() {
int64_t et = taosGetTimestampMs(); int64_t et = taosGetTimestampMs();
printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st); printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st);
assert(tSkipListGetSize(pSkipList) == size); assert(SL_GET_SIZE(pSkipList) == size);
tSkipListDestroy(pSkipList); tSkipListDestroy(pSkipList);
taosTFree(total); taosTFree(total);

105
src/util/tests/trefTest.c
View File

@ -13,104 +13,117 @@
typedef struct { typedef struct {
int refNum; int refNum;
int steps; int steps;
int refId; int rsetId;
int64_t rid;
void **p; void **p;
} SRefSpace; } SRefSpace;
void iterateRefs(int refId) { void iterateRefs(int rsetId) {
int count = 0; int count = 0;
void *p = taosIterateRef(refId, NULL); void *p = taosIterateRef(rsetId, NULL);
while (p) { while (p) {
// process P // process P
count++; count++;
p = taosIterateRef(refId, p); p = taosIterateRef(rsetId, p);
} }
printf(" %d ", count); printf(" %d ", count);
} }
void *takeRefActions(void *param) { void *addRef(void *param) {
SRefSpace *pSpace = (SRefSpace *)param; SRefSpace *pSpace = (SRefSpace *)param;
int code, id; int id;
int64_t rid;
for (int i=0; i < pSpace->steps; ++i) { for (int i=0; i < pSpace->steps; ++i) {
printf("s"); printf("a");
id = random() % pSpace->refNum; id = random() % pSpace->refNum;
code = taosAddRef(pSpace->refId, pSpace->p[id]); if (pSpace->rid[id] <= 0) {
usleep(1); pSpace->p[id] = malloc(128);
pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]);
}
usleep(100);
}
return NULL;
}
void *removeRef(void *param) {
SRefSpace *pSpace = (SRefSpace *)param;
int id;
int64_t rid;
for (int i=0; i < pSpace->steps; ++i) {
printf("d");
id = random() % pSpace->refNum;
if (pSpace->rid[id] > 0) {
code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]);
if (code == 0) pSpace->rid[id] = 0;
}
usleep(100);
}
return NULL;
}
void *acquireRelease(void *param) {
SRefSpace *pSpace = (SRefSpace *)param;
int id;
int64_t rid;
for (int i=0; i < pSpace->steps; ++i) {
printf("a");
id = random() % pSpace->refNum; id = random() % pSpace->refNum;
code = taosAcquireRef(pSpace->refId, pSpace->p[id]); code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]);
if (code >= 0) { if (code >= 0) {
usleep(id % 5 + 1); usleep(id % 5 + 1);
taosReleaseRef(pSpace->refId, pSpace->p[id]); taosReleaseRef(pSpace->rsetId, pSpace->p[id]);
} }
id = random() % pSpace->refNum;
taosRemoveRef(pSpace->refId, pSpace->p[id]);
usleep(id %5 + 1);
id = random() % pSpace->refNum;
code = taosAcquireRef(pSpace->refId, pSpace->p[id]);
if (code >= 0) {
usleep(id % 5 + 1);
taosReleaseRef(pSpace->refId, pSpace->p[id]);
} }
id = random() % pSpace->refNum;
iterateRefs(id);
}
for (int i=0; i < pSpace->refNum; ++i) {
taosRemoveRef(pSpace->refId, pSpace->p[i]);
}
//uInfo("refId:%d thread exits", pSpace->refId);
return NULL; return NULL;
} }
void myfree(void *p) { void myfree(void *p) {
return; free(p);
} }
void *openRefSpace(void *param) { void *openRefSpace(void *param) {
SRefSpace *pSpace = (SRefSpace *)param; SRefSpace *pSpace = (SRefSpace *)param;
printf("c"); printf("c");
pSpace->refId = taosOpenRef(50, myfree); pSpace->rsetId = taosOpenRef(50, myfree);
if (pSpace->refId < 0) { if (pSpace->rsetId < 0) {
printf("failed to open ref, reson:%s\n", tstrerror(pSpace->refId)); printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId));
return NULL; return NULL;
} }
pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum);
for (int i=0; i<pSpace->refNum; ++i) {
pSpace->p[i] = (void *) malloc(128);
}
pthread_attr_t thattr; pthread_attr_t thattr;
pthread_attr_init(&thattr); pthread_attr_init(&thattr);
pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
pthread_t thread1, thread2, thread3; pthread_t thread1, thread2, thread3;
pthread_create(&(thread1), &thattr, takeRefActions, (void *)(pSpace)); pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace));
pthread_create(&(thread2), &thattr, takeRefActions, (void *)(pSpace)); pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace));
pthread_create(&(thread3), &thattr, takeRefActions, (void *)(pSpace)); pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace));
pthread_join(thread1, NULL); pthread_join(thread1, NULL);
pthread_join(thread2, NULL); pthread_join(thread2, NULL);
pthread_join(thread3, NULL); pthread_join(thread3, NULL);
taosCloseRef(pSpace->refId);
for (int i=0; i<pSpace->refNum; ++i) { for (int i=0; i<pSpace->refNum; ++i) {
free(pSpace->p[i]); taosRemoveRef(pSpace->rsetId, pSpace->rid[i]);
} }
uInfo("refId:%d main thread exit", pSpace->refId); taosCloseRef(pSpace->rsetId);
uInfo("rsetId:%d main thread exit", pSpace->rsetId);
free(pSpace->p); free(pSpace->p);
pSpace->p = NULL; pSpace->p = NULL;
@ -140,7 +153,7 @@ int main(int argc, char *argv[]) {
printf("\nusage: %s [options] \n", argv[0]); printf("\nusage: %s [options] \n", argv[0]);
printf(" [-n]: number of references, default: %d\n", refNum); printf(" [-n]: number of references, default: %d\n", refNum);
printf(" [-s]: steps to run for each reference, default: %d\n", steps); printf(" [-s]: steps to run for each reference, default: %d\n", steps);
printf(" [-t]: number of refIds running in parallel, default: %d\n", threads); printf(" [-t]: number of rsetIds running in parallel, default: %d\n", threads);
printf(" [-l]: number of loops, default: %d\n", loops); printf(" [-l]: number of loops, default: %d\n", loops);
printf(" [-d]: debugFlag, default: %d\n", uDebugFlag); printf(" [-d]: debugFlag, default: %d\n", uDebugFlag);
exit(0); exit(0);

2
src/vnode/inc/vnodeInt.h
View File

@ -47,7 +47,7 @@ typedef struct {
void *rqueue; void *rqueue;
void *wal; void *wal;
void *tsdb; void *tsdb;
void *sync; int64_t sync;
void *events; void *events;
void *cq; // continuous query void *cq; // continuous query
int32_t cfgVersion; int32_t cfgVersion;

21
src/vnode/src/vnodeMain.c
View File

@ -38,12 +38,12 @@ static void vnodeCtrlFlow(void *handle, int32_t mseconds);
static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
#ifndef _SYNC #ifndef _SYNC
tsync_h syncStart(const SSyncInfo *info) { return NULL; } int64_t syncStart(const SSyncInfo *info) { return NULL; }
int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; } int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; }
void syncStop(tsync_h shandle) {} void syncStop(int64_t rid) {}
int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; } int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; }
int syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; } int syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; }
void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {} void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {}
#endif #endif
char* vnodeStatus[] = { char* vnodeStatus[] = {
@ -134,6 +134,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
tsdbCfg.precision = pVnodeCfg->cfg.precision; tsdbCfg.precision = pVnodeCfg->cfg.precision;
tsdbCfg.compression = pVnodeCfg->cfg.compression; tsdbCfg.compression = pVnodeCfg->cfg.compression;
tsdbCfg.update = pVnodeCfg->cfg.update;
char tsdbDir[TSDB_FILENAME_LEN] = {0}; char tsdbDir[TSDB_FILENAME_LEN] = {0};
sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
@ -324,7 +325,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
#ifndef _SYNC #ifndef _SYNC
pVnode->role = TAOS_SYNC_ROLE_MASTER; pVnode->role = TAOS_SYNC_ROLE_MASTER;
#else #else
if (pVnode->sync == NULL) { if (pVnode->sync <= 0) {
vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
tstrerror(terrno)); tstrerror(terrno));
vnodeCleanUp(pVnode); vnodeCleanUp(pVnode);
@ -554,9 +555,9 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
} }
// stop replication module // stop replication module
if (pVnode->sync) { if (pVnode->sync > 0) {
void *sync = pVnode->sync; int64_t sync = pVnode->sync;
pVnode->sync = NULL; pVnode->sync = -1;
syncStop(sync); syncStop(sync);
} }

2
src/vnode/src/vnodeRead.c
View File

@ -329,7 +329,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
} }
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
vDebug("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle); vError("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle);
vnodeBuildNoResultQueryRsp(pRet); vnodeBuildNoResultQueryRsp(pRet);
return code; return code;
} }

1
src/wal/inc/walInt.h
View File

@ -43,6 +43,7 @@ extern int32_t wDebugFlag;
typedef struct { typedef struct {
uint64_t version; uint64_t version;
int64_t fileId; int64_t fileId;
int64_t rid;
int32_t vgId; int32_t vgId;
int32_t fd; int32_t fd;
int32_t keep; int32_t keep;

9
src/wal/src/walMgmt.c
View File

@ -78,7 +78,8 @@ void *walOpen(char *path, SWalCfg *pCfg) {
return NULL; return NULL;
} }
if (taosAddRef(tsWal.refId, pWal) != TSDB_CODE_SUCCESS) { pWal->rid = taosAddRef(tsWal.refId, pWal);
if (pWal->rid < 0) {
walFreeObj(pWal); walFreeObj(pWal);
return NULL; return NULL;
} }
@ -143,7 +144,7 @@ void walClose(void *handle) {
} }
pthread_mutex_unlock(&pWal->mutex); pthread_mutex_unlock(&pWal->mutex);
taosRemoveRef(tsWal.refId, pWal); taosRemoveRef(tsWal.refId, pWal->rid);
} }
static int32_t walInitObj(SWal *pWal) { static int32_t walInitObj(SWal *pWal) {
@ -185,7 +186,7 @@ static void walUpdateSeq() {
} }
static void walFsyncAll() { static void walFsyncAll() {
SWal *pWal = taosIterateRef(tsWal.refId, NULL); SWal *pWal = taosIterateRef(tsWal.refId, 0);
while (pWal) { while (pWal) {
if (walNeedFsync(pWal)) { if (walNeedFsync(pWal)) {
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq); wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq);
@ -194,7 +195,7 @@ static void walFsyncAll() {
wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code)); wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code));
} }
} }
pWal = taosIterateRef(tsWal.refId, pWal); pWal = taosIterateRef(tsWal.refId, pWal->rid);
} }
} }

77
tests/examples/c/apitest.c
View File

@ -9,26 +9,40 @@
static void prepare_data(TAOS* taos) { static void prepare_data(TAOS* taos) {
taos_query(taos, "drop database if exists test;"); TAOS_RES *result;
result = taos_query(taos, "drop database if exists test;");
taos_free_result(result);
usleep(100000); usleep(100000);
taos_query(taos, "create database test;"); result = taos_query(taos, "create database test;");
taos_free_result(result);
usleep(100000); usleep(100000);
taos_select_db(taos, "test"); taos_select_db(taos, "test");
taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); result = taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);");
taos_free_result(result);
taos_query(taos, "create table t0 using meters tags(0);"); result = taos_query(taos, "create table t0 using meters tags(0);");
taos_query(taos, "create table t1 using meters tags(1);"); taos_free_result(result);
taos_query(taos, "create table t2 using meters tags(2);"); result = taos_query(taos, "create table t1 using meters tags(1);");
taos_query(taos, "create table t3 using meters tags(3);"); taos_free_result(result);
taos_query(taos, "create table t4 using meters tags(4);"); result = taos_query(taos, "create table t2 using meters tags(2);");
taos_query(taos, "create table t5 using meters tags(5);"); taos_free_result(result);
taos_query(taos, "create table t6 using meters tags(6);"); result = taos_query(taos, "create table t3 using meters tags(3);");
taos_query(taos, "create table t7 using meters tags(7);"); taos_free_result(result);
taos_query(taos, "create table t8 using meters tags(8);"); result = taos_query(taos, "create table t4 using meters tags(4);");
taos_query(taos, "create table t9 using meters tags(9);"); taos_free_result(result);
result = taos_query(taos, "create table t5 using meters tags(5);");
taos_free_result(result);
result = taos_query(taos, "create table t6 using meters tags(6);");
taos_free_result(result);
result = taos_query(taos, "create table t7 using meters tags(7);");
taos_free_result(result);
result = taos_query(taos, "create table t8 using meters tags(8);");
taos_free_result(result);
result = taos_query(taos, "create table t9 using meters tags(9);");
taos_free_result(result);
TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)" result = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)"
" ('2020-01-01 00:01:00.000', 0)" " ('2020-01-01 00:01:00.000', 0)"
" ('2020-01-01 00:02:00.000', 0)" " ('2020-01-01 00:02:00.000', 0)"
" t1 values('2020-01-01 00:00:00.000', 0)" " t1 values('2020-01-01 00:00:00.000', 0)"
@ -46,10 +60,11 @@ static void prepare_data(TAOS* taos) {
" t7 values('2020-01-01 00:01:02.000', 0)" " t7 values('2020-01-01 00:01:02.000', 0)"
" t8 values('2020-01-01 00:01:02.000', 0)" " t8 values('2020-01-01 00:01:02.000', 0)"
" t9 values('2020-01-01 00:01:02.000', 0)"); " t9 values('2020-01-01 00:01:02.000', 0)");
int affected = taos_affected_rows(res); int affected = taos_affected_rows(result);
if (affected != 18) { if (affected != 18) {
printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected); printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected);
} }
taos_free_result(result);
// super tables subscription // super tables subscription
usleep(1000000); usleep(1000000);
} }
@ -135,6 +150,7 @@ static void verify_query(TAOS* taos) {
res = taos_query(taos, "select * from meters"); res = taos_query(taos, "select * from meters");
taos_stop_query(res); taos_stop_query(res);
taos_free_result(res);
} }
@ -153,23 +169,30 @@ static void verify_subscribe(TAOS* taos) {
res = taos_consume(tsub); res = taos_consume(tsub);
check_row_count(__LINE__, res, 0); check_row_count(__LINE__, res, 0);
taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); TAOS_RES *result;
taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); result = taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);");
taos_free_result(result);
result = taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);");
taos_free_result(result);
res = taos_consume(tsub); res = taos_consume(tsub);
check_row_count(__LINE__, res, 2); check_row_count(__LINE__, res, 2);
taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); result = taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);");
taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); taos_free_result(result);
result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);");
taos_free_result(result);
res = taos_consume(tsub); res = taos_consume(tsub);
check_row_count(__LINE__, res, 2); check_row_count(__LINE__, res, 2);
taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);");
taos_free_result(result);
res = taos_consume(tsub); res = taos_consume(tsub);
check_row_count(__LINE__, res, 1); check_row_count(__LINE__, res, 1);
// keep progress information and restart subscription // keep progress information and restart subscription
taos_unsubscribe(tsub, 1); taos_unsubscribe(tsub, 1);
taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);");
taos_free_result(result);
tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0); tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0);
res = taos_consume(tsub); res = taos_consume(tsub);
check_row_count(__LINE__, res, 24); check_row_count(__LINE__, res, 24);
@ -196,7 +219,8 @@ static void verify_subscribe(TAOS* taos) {
res = taos_consume(tsub); res = taos_consume(tsub);
check_row_count(__LINE__, res, 0); check_row_count(__LINE__, res, 0);
taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);");
taos_free_result(result);
res = taos_consume(tsub); res = taos_consume(tsub);
check_row_count(__LINE__, res, 1); check_row_count(__LINE__, res, 1);
@ -205,7 +229,8 @@ static void verify_subscribe(TAOS* taos) {
int blockFetch = 0; int blockFetch = 0;
tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000); tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000);
usleep(2000000); usleep(2000000);
taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); result = taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);");
taos_free_result(result);
usleep(2000000); usleep(2000000);
taos_unsubscribe(tsub, 0); taos_unsubscribe(tsub, 0);
} }
@ -213,8 +238,9 @@ static void verify_subscribe(TAOS* taos) {
void verify_prepare(TAOS* taos) { void verify_prepare(TAOS* taos) {
TAOS_RES* result = taos_query(taos, "drop database if exists test;"); TAOS_RES* result = taos_query(taos, "drop database if exists test;");
taos_free_result(result);
usleep(100000); usleep(100000);
taos_query(taos, "create database test;"); result = taos_query(taos, "create database test;");
int code = taos_errno(result); int code = taos_errno(result);
if (code != 0) { if (code != 0) {
@ -429,7 +455,8 @@ void verify_stream(TAOS* taos) {
NULL); NULL);
printf("waiting for stream data\n"); printf("waiting for stream data\n");
usleep(100000); usleep(100000);
taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); TAOS_RES* result = taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);");
taos_free_result(result);
usleep(200000000); usleep(200000000);
taos_close_stream(strm); taos_close_stream(strm);
} }

41
tests/examples/c/asyncdemo.c
View File

@ -46,6 +46,34 @@ void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
void taos_select_call_back(void *param, TAOS_RES *tres, int code); void taos_select_call_back(void *param, TAOS_RES *tres, int code);
void taos_error(TAOS *taos); void taos_error(TAOS *taos);
static void queryDB(TAOS *taos, char *command) {
int i;
TAOS_RES *pSql = NULL;
int32_t code = -1;
for (i = 0; i < 5; i++) {
if (NULL != pSql) {
taos_free_result(pSql);
pSql = NULL;
}
pSql = taos_query(taos, command);
code = taos_errno(pSql);
if (0 == code) {
break;
}
}
if (code != 0) {
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
taos_free_result(pSql);
taos_close(taos);
exit(EXIT_FAILURE);
}
taos_free_result(pSql);
}
int main(int argc, char *argv[]) int main(int argc, char *argv[])
{ {
TAOS *taos; TAOS *taos;
@ -78,16 +106,14 @@ int main(int argc, char *argv[])
printf("success to connect to server\n"); printf("success to connect to server\n");
sprintf(sql, "drop database %s", db); sprintf(sql, "drop database if exists %s", db);
taos_query(taos, sql); queryDB(taos, sql);
sprintf(sql, "create database %s", db); sprintf(sql, "create database %s", db);
if (taos_query(taos, sql) != 0) queryDB(taos, sql);
taos_error(taos);
sprintf(sql, "use %s", db); sprintf(sql, "use %s", db);
if (taos_query(taos, sql) != 0) queryDB(taos, sql);
taos_error(taos);
strcpy(prefix, "asytbl_"); strcpy(prefix, "asytbl_");
for (i = 0; i < numOfTables; ++i) { for (i = 0; i < numOfTables; ++i) {
@ -95,8 +121,7 @@ int main(int argc, char *argv[])
tableList[i].taos = taos; tableList[i].taos = taos;
sprintf(tableList[i].name, "%s%d", prefix, i); sprintf(tableList[i].name, "%s%d", prefix, i);
sprintf(sql, "create table %s%d (ts timestamp, volume bigint)", prefix, i); sprintf(sql, "create table %s%d (ts timestamp, volume bigint)", prefix, i);
if (taos_query(taos, sql) != 0) queryDB(taos, sql);
taos_error(taos);
} }
gettimeofday(&systemTime, NULL); gettimeofday(&systemTime, NULL);

57
tests/examples/c/demo.c
View File

@ -22,6 +22,34 @@
#include <inttypes.h> #include <inttypes.h>
#include <taos.h> // TAOS header file #include <taos.h> // TAOS header file
static void queryDB(TAOS *taos, char *command) {
int i;
TAOS_RES *pSql = NULL;
int32_t code = -1;
for (i = 0; i < 5; i++) {
if (NULL != pSql) {
taos_free_result(pSql);
pSql = NULL;
}
pSql = taos_query(taos, command);
code = taos_errno(pSql);
if (0 == code) {
break;
}
}
if (code != 0) {
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
taos_free_result(pSql);
taos_close(taos);
exit(EXIT_FAILURE);
}
taos_free_result(pSql);
}
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
TAOS * taos; TAOS * taos;
char qstr[1024]; char qstr[1024];
@ -44,22 +72,26 @@ int main(int argc, char *argv[]) {
printf("success to connect to server\n"); printf("success to connect to server\n");
taos_query(taos, "drop database demo"); //taos_query(taos, "drop database demo");
queryDB(taos, "drop database if exists demo");
result = taos_query(taos, "create database demo"); //result = taos_query(taos, "create database demo");
if (result == NULL) { //if (result == NULL) {
printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/); // printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/);
exit(1); // exit(1);
} //}
queryDB(taos, "create database demo");
printf("success to create database\n"); printf("success to create database\n");
taos_query(taos, "use demo"); //taos_query(taos, "use demo");
queryDB(taos, "use demo");
// create table // create table
if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) { //if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) {
printf("failed to create table, reason:%s\n", taos_errstr(result)); // printf("failed to create table, reason:%s\n", taos_errstr(result));
exit(1); // exit(1);
} //}
queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))");
printf("success to create table\n"); printf("success to create table\n");
// sleep for one second to make sure table is created on data node // sleep for one second to make sure table is created on data node
@ -80,8 +112,10 @@ int main(int argc, char *argv[]) {
printf("insert row: %i\n", i); printf("insert row: %i\n", i);
} else { } else {
printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/); printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/);
taos_free_result(result);
exit(1); exit(1);
} }
taos_free_result(result);
//sleep(1); //sleep(1);
} }
@ -92,6 +126,7 @@ int main(int argc, char *argv[]) {
result = taos_query(taos, qstr); result = taos_query(taos, qstr);
if (result == NULL || taos_errno(result) != 0) { if (result == NULL || taos_errno(result) != 0) {
printf("failed to select, reason:%s\n", taos_errstr(result)); printf("failed to select, reason:%s\n", taos_errstr(result));
taos_free_result(result);
exit(1); exit(1);
} }

71
tests/pytest/alter/db_update_options.py Normal file
View File

@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
import random
import string
import subprocess
import sys
from util.log import *
from util.cases import *
from util.sql import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
tdLog.debug("check database")
tdSql.prepare()
# check default update value
sql = "create database if not exists db"
tdSql.execute(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,0)
sql = "alter database db update 1"
# check update value
tdSql.execute(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,1)
sql = "alter database db update 0"
tdSql.execute(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,0)
sql = "alter database db update -1"
tdSql.error(sql)
sql = "alter database db update 100"
tdSql.error(sql)
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,0)
tdSql.execute('drop database db')
tdSql.error('create database db update 100')
tdSql.error('create database db update -1')
tdSql.execute('create database db update 1')
tdSql.query('show databases')
tdSql.checkRows(1)
tdSql.checkData(0,16,1)
tdSql.execute('drop database db')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

227
tests/pytest/crash_gen/crash_gen.py → tests/pytest/crash_gen/crash_gen_main.py
View File

@ -38,9 +38,9 @@ import resource
from guppy import hpy from guppy import hpy
import gc import gc
from .service_manager import ServiceManager, TdeInstance from crash_gen.service_manager import ServiceManager, TdeInstance
from .misc import Logging, Status, CrashGenError, Dice, Helper, Progress from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress
from .db import DbConn, MyTDSql, DbConnNative, DbManager from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager
import taos import taos
import requests import requests
@ -243,7 +243,7 @@ class WorkerThread:
class ThreadCoordinator: class ThreadCoordinator:
WORKER_THREAD_TIMEOUT = 180 # one minute WORKER_THREAD_TIMEOUT = 120 # Normal: 120
def __init__(self, pool: ThreadPool, dbManager: DbManager): def __init__(self, pool: ThreadPool, dbManager: DbManager):
self._curStep = -1 # first step is 0 self._curStep = -1 # first step is 0
@ -388,9 +388,9 @@ class ThreadCoordinator:
self._syncAtBarrier() # For now just cross the barrier self._syncAtBarrier() # For now just cross the barrier
Progress.emit(Progress.END_THREAD_STEP) Progress.emit(Progress.END_THREAD_STEP)
except threading.BrokenBarrierError as err: except threading.BrokenBarrierError as err:
Logging.info("Main loop aborted, caused by worker thread time-out") Logging.info("Main loop aborted, caused by worker thread(s) time-out")
self._execStats.registerFailure("Aborted due to worker thread timeout") self._execStats.registerFailure("Aborted due to worker thread timeout")
print("\n\nWorker Thread time-out detected, important thread info:") print("\n\nWorker Thread time-out detected, TAOS related threads are:")
ts = ThreadStacks() ts = ThreadStacks()
ts.print(filterInternal=True) ts.print(filterInternal=True)
workerTimeout = True workerTimeout = True
@ -435,7 +435,7 @@ class ThreadCoordinator:
Logging.debug("\r\n\n--> Main thread ready to finish up...") Logging.debug("\r\n\n--> Main thread ready to finish up...")
Logging.debug("Main thread joining all threads") Logging.debug("Main thread joining all threads")
self._pool.joinAll() # Get all threads to finish self._pool.joinAll() # Get all threads to finish
Logging.info("\nAll worker threads finished") Logging.info(". . . All worker threads finished") # No CR/LF before
self._execStats.endExec() self._execStats.endExec()
def cleanup(self): # free resources def cleanup(self): # free resources
@ -1072,17 +1072,18 @@ class Database:
t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years
t4 = datetime.datetime.fromtimestamp( t4 = datetime.datetime.fromtimestamp(
t3.timestamp() + elSec2) # see explanation above t3.timestamp() + elSec2) # see explanation above
Logging.info("Setting up TICKS to start from: {}".format(t4)) Logging.debug("Setting up TICKS to start from: {}".format(t4))
return t4 return t4
@classmethod @classmethod
def getNextTick(cls): def getNextTick(cls):
with cls._clsLock: # prevent duplicate tick with cls._clsLock: # prevent duplicate tick
if cls._lastLaggingTick==0: if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized
# 10k at 1/20 chance, should be enough to avoid overlaps # 10k at 1/20 chance, should be enough to avoid overlaps
cls._lastLaggingTick = cls.setupLastTick() + datetime.timedelta(0, -10000) tick = cls.setupLastTick()
if cls._lastTick==0: # should be quite a bit into the future cls._lastTick = tick
cls._lastTick = cls.setupLastTick() cls._lastLaggingTick = tick + datetime.timedelta(0, -10000)
# if : # should be quite a bit into the future
if Dice.throw(20) == 0: # 1 in 20 chance, return lagging tick if Dice.throw(20) == 0: # 1 in 20 chance, return lagging tick
cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds
@ -1177,6 +1178,8 @@ class Task():
instead. But a task is always associated with a DB instead. But a task is always associated with a DB
''' '''
taskSn = 100 taskSn = 100
_lock = threading.Lock()
_tableLocks: Dict[str, threading.Lock] = {}
@classmethod @classmethod
def allocTaskNum(cls): def allocTaskNum(cls):
@ -1198,6 +1201,8 @@ class Task():
self._execStats = execStats self._execStats = execStats
self._db = db # A task is always associated/for a specific DB self._db = db # A task is always associated/for a specific DB
def isSuccess(self): def isSuccess(self):
return self._err is None return self._err is None
@ -1237,6 +1242,7 @@ class Task():
0x0B, # Unable to establish connection, more details in TD-1648 0x0B, # Unable to establish connection, more details in TD-1648
0x200, # invalid SQL， TODO: re-examine with TD-934 0x200, # invalid SQL， TODO: re-examine with TD-934
0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776 0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776
0x213, # "Disconnected from service", result of "kill connection ???"
0x217, # "db not selected", client side defined error code 0x217, # "db not selected", client side defined error code
# 0x218, # "Table does not exist" client side defined error code # 0x218, # "Table does not exist" client side defined error code
0x360, # Table already exists 0x360, # Table already exists
@ -1318,7 +1324,7 @@ class Task():
self._err = err self._err = err
self._aborted = True self._aborted = True
except Exception as e: except Exception as e:
self.logInfo("Non-TAOS exception encountered") Logging.info("Non-TAOS exception encountered with: {}".format(self.__class__.__name__))
self._err = e self._err = e
self._aborted = True self._aborted = True
traceback.print_exc() traceback.print_exc()
@ -1351,6 +1357,24 @@ class Task():
def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread
return wt.getQueryResult() return wt.getQueryResult()
def lockTable(self, ftName): # full table name
# print(" <<" + ftName + '_', end="", flush=True)
with Task._lock:
if not ftName in Task._tableLocks:
Task._tableLocks[ftName] = threading.Lock()
Task._tableLocks[ftName].acquire()
def unlockTable(self, ftName):
# print('_' + ftName + ">> ", end="", flush=True)
with Task._lock:
if not ftName in self._tableLocks:
raise RuntimeError("Corrupt state, no such lock")
lock = Task._tableLocks[ftName]
if not lock.locked():
raise RuntimeError("Corrupte state, already unlocked")
lock.release()
class ExecutionStats: class ExecutionStats:
def __init__(self): def __init__(self):
@ -1461,7 +1485,7 @@ class StateTransitionTask(Task):
_baseTableNumber = None _baseTableNumber = None
_endState = None _endState = None # TODO: no longter used?
@classmethod @classmethod
def getInfo(cls): # each sub class should supply their own information def getInfo(cls): # each sub class should supply their own information
@ -1486,7 +1510,7 @@ class StateTransitionTask(Task):
@classmethod @classmethod
def getRegTableName(cls, i): def getRegTableName(cls, i):
if ( StateTransitionTask._baseTableNumber is None): if ( StateTransitionTask._baseTableNumber is None): # Set it one time
StateTransitionTask._baseTableNumber = Dice.throw( StateTransitionTask._baseTableNumber = Dice.throw(
999) if gConfig.dynamic_db_table_names else 0 999) if gConfig.dynamic_db_table_names else 0
return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i) return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i)
@ -1544,8 +1568,11 @@ class TaskCreateSuperTable(StateTransitionTask):
sTable = self._db.getFixedSuperTable() # type: TdSuperTable sTable = self._db.getFixedSuperTable() # type: TdSuperTable
# wt.execSql("use db") # should always be in place # wt.execSql("use db") # should always be in place
sTable.create(wt.getDbConn(), self._db.getName(), sTable.create(wt.getDbConn(), self._db.getName(),
{'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'}) {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'},
dropIfExists = True
)
# self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName))
# No need to create the regular tables, INSERT will do that # No need to create the regular tables, INSERT will do that
# automatically # automatically
@ -1558,13 +1585,40 @@ class TdSuperTable:
def getName(self): def getName(self):
return self._stName return self._stName
def drop(self, dbc, dbName, skipCheck = False):
if self.exists(dbc, dbName) : # if myself exists
fullTableName = dbName + '.' + self._stName
dbc.execute("DROP TABLE {}".format(fullTableName))
else:
if not skipCheck:
raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName))
def exists(self, dbc, dbName):
dbc.execute("USE " + dbName)
return dbc.existsSuperTable(self._stName)
# TODO: odd semantic, create() method is usually static? # TODO: odd semantic, create() method is usually static?
def create(self, dbc, dbName, cols: dict, tags: dict): def create(self, dbc, dbName, cols: dict, tags: dict,
dropIfExists = False
):
'''Creating a super table''' '''Creating a super table'''
sql = "CREATE TABLE {}.{} ({}) TAGS ({})".format( dbc.execute("USE " + dbName)
dbName, fullTableName = dbName + '.' + self._stName
self._stName, if dbc.existsSuperTable(self._stName):
",".join(['%s %s'%(k,v) for (k,v) in cols.items()]), if dropIfExists:
dbc.execute("DROP TABLE {}".format(fullTableName))
else: # error
raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName))
# Now let's create
sql = "CREATE TABLE {} ({})".format(
fullTableName,
",".join(['%s %s'%(k,v) for (k,v) in cols.items()]))
if tags is None :
sql += " TAGS (dummy int) "
else:
sql += " TAGS ({})".format(
",".join(['%s %s'%(k,v) for (k,v) in tags.items()]) ",".join(['%s %s'%(k,v) for (k,v) in tags.items()])
) )
dbc.execute(sql) dbc.execute(sql)
@ -1583,14 +1637,25 @@ class TdSuperTable:
def hasRegTables(self, dbc: DbConn, dbName: str): def hasRegTables(self, dbc: DbConn, dbName: str):
return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0 return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0
def ensureTable(self, dbc: DbConn, dbName: str, regTableName: str): def ensureTable(self, task: Task, dbc: DbConn, dbName: str, regTableName: str):
sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName) sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName)
if dbc.query(sql) >= 1 : # reg table exists already if dbc.query(sql) >= 1 : # reg table exists already
return return
sql = "CREATE TABLE {}.{} USING {}.{} tags ({})".format(
dbName, regTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) # acquire a lock first, so as to be able to *verify*. More details in TD-1471
fullTableName = dbName + '.' + regTableName
if task is not None: # optional lock
task.lockTable(fullTableName)
Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table
# print("(" + fullTableName[-3:] + ")", end="", flush=True)
try:
sql = "CREATE TABLE {} USING {}.{} tags ({})".format(
fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName)
) )
dbc.execute(sql) dbc.execute(sql)
finally:
if task is not None:
task.unlockTable(fullTableName) # no matter what
def _getTagStrForSql(self, dbc, dbName: str) : def _getTagStrForSql(self, dbc, dbName: str) :
tags = self._getTags(dbc, dbName) tags = self._getTags(dbc, dbName)
@ -1809,7 +1874,7 @@ class TaskRestartService(StateTransitionTask):
with self._classLock: with self._classLock:
if self._isRunning: if self._isRunning:
print("Skipping restart task, another running already") Logging.info("Skipping restart task, another running already")
return return
self._isRunning = True self._isRunning = True
@ -1847,24 +1912,21 @@ class TaskAddData(StateTransitionTask):
def canBeginFrom(cls, state: AnyState): def canBeginFrom(cls, state: AnyState):
return state.canAddData() return state.canAddData()
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): def _addDataInBatch(self, db, dbc, regTableName, te: TaskExecutor):
# ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
db = self._db fullTableName = db.getName() + '.' + regTableName
dbc = wt.getDbConn()
tblSeq = list(range(
self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES))
random.shuffle(tblSeq)
for i in tblSeq:
if (i in self.activeTable): # wow already active
print("x", end="", flush=True) # concurrent insertion
else:
self.activeTable.add(i) # marking it active
sTable = db.getFixedSuperTable() sql = "insert into {} values ".format(fullTableName)
regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) for j in range(numRecords): # number of records per table
sTable.ensureTable(wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists nextInt = db.getNextInt()
nextTick = db.getNextTick()
sql += "('{}', {});".format(nextTick, nextInt)
dbc.execute(sql)
for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table def _addData(self, db, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches
numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
for j in range(numRecords): # number of records per table
nextInt = db.getNextInt() nextInt = db.getNextInt()
nextTick = db.getNextTick() nextTick = db.getNextTick()
if gConfig.record_ops: if gConfig.record_ops:
@ -1872,22 +1934,24 @@ class TaskAddData(StateTransitionTask):
self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName))
self.fAddLogReady.flush() self.fAddLogReady.flush()
os.fsync(self.fAddLogReady) os.fsync(self.fAddLogReady)
sql = "insert into {}.{} values ('{}', {});".format( # removed: tags ('{}', {})
db.getName(), # TODO: too ugly trying to lock the table reliably, refactor...
regTableName, fullTableName = db.getName() + '.' + regTableName
if gConfig.verify_data:
self.lockTable(fullTableName)
# print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written
try:
sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {})
fullTableName,
# ds.getFixedSuperTableName(), # ds.getFixedSuperTableName(),
# ds.getNextBinary(), ds.getNextFloat(), # ds.getNextBinary(), ds.getNextFloat(),
nextTick, nextInt) nextTick, nextInt)
dbc.execute(sql) dbc.execute(sql)
# Successfully wrote the data into the DB, let's record it except: # Any exception at all
# somehow if gConfig.verify_data:
te.recordDataMark(nextInt) self.unlockTable(fullTableName)
if gConfig.record_ops: raise
self.fAddLogDone.write(
"Wrote {} to {}\n".format(
nextInt, regTableName))
self.fAddLogDone.flush()
os.fsync(self.fAddLogDone)
# Now read it back and verify, we might encounter an error if table is dropped # Now read it back and verify, we might encounter an error if table is dropped
if gConfig.verify_data: # only if command line asks for it if gConfig.verify_data: # only if command line asks for it
@ -1905,16 +1969,52 @@ class TaskAddData(StateTransitionTask):
"Failed to read back same data for tick: {}, wrote: {}, read: {}" "Failed to read back same data for tick: {}, wrote: {}, read: {}"
.format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"),
errno) errno)
# Re-throw no matter what elif errno in [0x218, 0x362]: # table doesn't exist
# do nothing
dummy = 0
else:
# Re-throw otherwise
raise raise
finally:
self.unlockTable(fullTableName) # Unlock the table no matter what
# Successfully wrote the data into the DB, let's record it somehow
te.recordDataMark(nextInt)
if gConfig.record_ops:
self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName))
self.fAddLogDone.flush()
os.fsync(self.fAddLogDone)
def _executeInternal(self, te: TaskExecutor, wt: WorkerThread):
# ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access
db = self._db
dbc = wt.getDbConn()
numTables = self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES
numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS
tblSeq = list(range(numTables ))
random.shuffle(tblSeq) # now we have random sequence
for i in tblSeq:
if (i in self.activeTable): # wow already active
print("x", end="", flush=True) # concurrent insertion
else:
self.activeTable.add(i) # marking it active
sTable = db.getFixedSuperTable()
regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i)
fullTableName = db.getName() + '.' + regTableName
# self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked"
sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists
# self._unlockTable(fullTableName)
if Dice.throw(1) == 0: # 1 in 2 chance
self._addData(db, dbc, regTableName, te)
else:
self._addDataInBatch(db, dbc, regTableName, te)
self.activeTable.discard(i) # not raising an error, unlike remove self.activeTable.discard(i) # not raising an error, unlike remove
class ThreadStacks: # stack info for all threads class ThreadStacks: # stack info for all threads
def __init__(self): def __init__(self):
self._allStacks = {} self._allStacks = {}
@ -1936,17 +2036,18 @@ class ThreadStacks: # stack info for all threads
'__init__']: # the thread that extracted the stack '__init__']: # the thread that extracted the stack
continue # ignore continue # ignore
# Now print # Now print
print("\n<----- Thread Info for ID: {}".format(thNid)) print("\n<----- Thread Info for LWP/ID: {} (Execution stopped at Bottom Frame) <-----".format(thNid))
stackFrame = 0
for frame in stack: for frame in stack:
# print(frame) # print(frame)
print("File {filename}, line {lineno}, in {name}".format( print("[{sf}] File {filename}, line {lineno}, in {name}".format(
filename=frame.filename, lineno=frame.lineno, name=frame.name)) sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name))
print(" {}".format(frame.line)) print(" {}".format(frame.line))
print("-----> End of Thread Info\n") print("-----> End of Thread Info ----->\n")
class ClientManager: class ClientManager:
def __init__(self): def __init__(self):
print("Starting service manager") Logging.info("Starting service manager")
# signal.signal(signal.SIGTERM, self.sigIntHandler) # signal.signal(signal.SIGTERM, self.sigIntHandler)
# signal.signal(signal.SIGINT, self.sigIntHandler) # signal.signal(signal.SIGINT, self.sigIntHandler)
@ -2048,7 +2149,7 @@ class ClientManager:
thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps) thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps)
self.tc = ThreadCoordinator(thPool, dbManager) self.tc = ThreadCoordinator(thPool, dbManager)
print("Starting client instance to: {}".format(tInst)) Logging.info("Starting client instance: {}".format(tInst))
self.tc.run() self.tc.run()
# print("exec stats: {}".format(self.tc.getExecStats())) # print("exec stats: {}".format(self.tc.getExecStats()))
# print("TC failed = {}".format(self.tc.isFailed())) # print("TC failed = {}".format(self.tc.isFailed()))

6
tests/pytest/crash_gen/db.py
View File

@ -95,6 +95,11 @@ class DbConn:
# print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName))) # print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName)))
return dbName in dbs # TODO: super weird type mangling seen, once here return dbName in dbs # TODO: super weird type mangling seen, once here
def existsSuperTable(self, stName):
self.query("show stables")
sts = [v[0] for v in self.getQueryResult()]
return stName in sts
def hasTables(self): def hasTables(self):
return self.query("show tables") > 0 return self.query("show tables") > 0
@ -240,6 +245,7 @@ class MyTDSql:
def _execInternal(self, sql): def _execInternal(self, sql):
startTime = time.time() startTime = time.time()
# Logging.debug("Executing SQL: " + sql)
ret = self._cursor.execute(sql) ret = self._cursor.execute(sql)
# print("\nSQL success: {}".format(sql)) # print("\nSQL success: {}".format(sql))
queryTime = time.time() - startTime queryTime = time.time() - startTime

9
tests/pytest/crash_gen/misc.py
View File

@ -27,7 +27,7 @@ class LoggingFilter(logging.Filter):
class MyLoggingAdapter(logging.LoggerAdapter): class MyLoggingAdapter(logging.LoggerAdapter):
def process(self, msg, kwargs): def process(self, msg, kwargs):
return "[{}] {}".format(threading.get_ident() % 10000, msg), kwargs return "[{:04d}] {}".format(threading.get_ident() % 10000, msg), kwargs
# return '[%s] %s' % (self.extra['connid'], msg), kwargs # return '[%s] %s' % (self.extra['connid'], msg), kwargs
@ -51,7 +51,7 @@ class Logging:
_logger.addHandler(ch) _logger.addHandler(ch)
# Logging adapter, to be used as a logger # Logging adapter, to be used as a logger
print("setting logger variable") # print("setting logger variable")
# global logger # global logger
cls.logger = MyLoggingAdapter(_logger, []) cls.logger = MyLoggingAdapter(_logger, [])
@ -166,6 +166,9 @@ class Progress:
SERVICE_RECONNECT_START = 4 SERVICE_RECONNECT_START = 4
SERVICE_RECONNECT_SUCCESS = 5 SERVICE_RECONNECT_SUCCESS = 5
SERVICE_RECONNECT_FAILURE = 6 SERVICE_RECONNECT_FAILURE = 6
SERVICE_START_NAP = 7
CREATE_TABLE_ATTEMPT = 8
tokens = { tokens = {
STEP_BOUNDARY: '.', STEP_BOUNDARY: '.',
BEGIN_THREAD_STEP: '[', BEGIN_THREAD_STEP: '[',
@ -174,6 +177,8 @@ class Progress:
SERVICE_RECONNECT_START: '<r.', SERVICE_RECONNECT_START: '<r.',
SERVICE_RECONNECT_SUCCESS: '.r>', SERVICE_RECONNECT_SUCCESS: '.r>',
SERVICE_RECONNECT_FAILURE: '.xr>', SERVICE_RECONNECT_FAILURE: '.xr>',
SERVICE_START_NAP: '_zz',
CREATE_TABLE_ATTEMPT: '_c',
} }
@classmethod @classmethod

81
tests/pytest/crash_gen/service_manager.py
View File

@ -47,6 +47,17 @@ class TdeInstance():
.format(selfPath, projPath)) .format(selfPath, projPath))
return buildPath return buildPath
@classmethod
def prepareGcovEnv(cls, env):
# Ref: https://gcc.gnu.org/onlinedocs/gcc/Cross-profiling.html
bPath = cls._getBuildPath() # build PATH
numSegments = len(bPath.split('/')) - 1 # "/x/TDengine/build" should yield 3
numSegments = numSegments - 1 # DEBUG only
env['GCOV_PREFIX'] = bPath + '/svc_gcov'
env['GCOV_PREFIX_STRIP'] = str(numSegments) # Strip every element, plus, ENV needs strings
Logging.info("Preparing GCOV environement to strip {} elements and use path: {}".format(
numSegments, env['GCOV_PREFIX'] ))
def __init__(self, subdir='test', tInstNum=0, port=6030, fepPort=6030): def __init__(self, subdir='test', tInstNum=0, port=6030, fepPort=6030):
self._buildDir = self._getBuildPath() self._buildDir = self._getBuildPath()
self._subdir = '/' + subdir # TODO: tolerate "/" self._subdir = '/' + subdir # TODO: tolerate "/"
@ -217,6 +228,11 @@ class TdeSubProcess:
# raise CrashGenError("Empty instance not allowed in TdeSubProcess") # raise CrashGenError("Empty instance not allowed in TdeSubProcess")
# self._tInst = tInst # Default create at ServiceManagerThread # self._tInst = tInst # Default create at ServiceManagerThread
def __repr__(self):
if self.subProcess is None:
return '[TdeSubProc: Empty]'
return '[TdeSubProc: pid = {}]'.format(self.getPid())
def getStdOut(self): def getStdOut(self):
return self.subProcess.stdout return self.subProcess.stdout
@ -236,16 +252,29 @@ class TdeSubProcess:
if self.subProcess: # already there if self.subProcess: # already there
raise RuntimeError("Corrupt process state") raise RuntimeError("Corrupt process state")
# Prepare environment variables for coverage information
# Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment
myEnv = os.environ.copy()
TdeInstance.prepareGcovEnv(myEnv)
# print(myEnv)
# print(myEnv.items())
# print("Starting TDengine via Shell: {}".format(cmdLineStr))
useShell = True
self.subProcess = subprocess.Popen( self.subProcess = subprocess.Popen(
cmdLine, ' '.join(cmdLine) if useShell else cmdLine,
shell=False, shell=useShell,
# svcCmdSingle, shell=True, # capture core dump? # svcCmdSingle, shell=True, # capture core dump?
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
# bufsize=1, # not supported in binary mode # bufsize=1, # not supported in binary mode
close_fds=ON_POSIX close_fds=ON_POSIX,
env=myEnv
) # had text=True, which interferred with reading EOF ) # had text=True, which interferred with reading EOF
STOP_SIGNAL = signal.SIGKILL # What signal to use (in kill) to stop a taosd process?
def stop(self): def stop(self):
""" """
Stop a sub process, and try to return a meaningful return code. Stop a sub process, and try to return a meaningful return code.
@ -267,7 +296,7 @@ class TdeSubProcess:
SIGUSR2 12 SIGUSR2 12
""" """
if not self.subProcess: if not self.subProcess:
print("Sub process already stopped") Logging.error("Sub process already stopped")
return # -1 return # -1
retCode = self.subProcess.poll() # ret -N means killed with signal N, otherwise it's from exit(N) retCode = self.subProcess.poll() # ret -N means killed with signal N, otherwise it's from exit(N)
@ -278,20 +307,25 @@ class TdeSubProcess:
return retCode return retCode
# process still alive, let's interrupt it # process still alive, let's interrupt it
print("Terminate running process, send SIG_INT and wait...") Logging.info("Terminate running process, send SIG_{} and wait...".format(self.STOP_SIGNAL))
# sub process should end, then IPC queue should end, causing IO thread to end # sub process should end, then IPC queue should end, causing IO thread to end
# sig = signal.SIGINT topSubProc = psutil.Process(self.subProcess.pid)
sig = signal.SIGKILL for child in topSubProc.children(recursive=True): # or parent.children() for recursive=False
self.subProcess.send_signal(sig) # SIGNINT or SIGKILL child.send_signal(self.STOP_SIGNAL)
time.sleep(0.2) # 200 ms
# topSubProc.send_signal(sig) # now kill the main sub process (likely the Shell)
self.subProcess.send_signal(self.STOP_SIGNAL) # main sub process (likely the Shell)
self.subProcess.wait(20) self.subProcess.wait(20)
retCode = self.subProcess.returncode # should always be there retCode = self.subProcess.returncode # should always be there
# May throw subprocess.TimeoutExpired exception above, therefore # May throw subprocess.TimeoutExpired exception above, therefore
# The process is guranteed to have ended by now # The process is guranteed to have ended by now
self.subProcess = None self.subProcess = None
if retCode != 0: # != (- signal.SIGINT): if retCode != 0: # != (- signal.SIGINT):
Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(sig, retCode)) Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(
self.STOP_SIGNAL, retCode))
else: else:
Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(sig)) Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(self.STOP_SIGNAL))
return - retCode return - retCode
class ServiceManager: class ServiceManager:
@ -439,7 +473,7 @@ class ServiceManager:
time.sleep(self.PAUSE_BETWEEN_IPC_CHECK) # pause, before next round time.sleep(self.PAUSE_BETWEEN_IPC_CHECK) # pause, before next round
# raise CrashGenError("dummy") # raise CrashGenError("dummy")
print("Service Manager Thread (with subprocess) ended, main thread exiting...") Logging.info("Service Manager Thread (with subprocess) ended, main thread exiting...")
def _getFirstInstance(self): def _getFirstInstance(self):
return self._tInsts[0] return self._tInsts[0]
@ -452,7 +486,7 @@ class ServiceManager:
# Find if there's already a taosd service, and then kill it # Find if there's already a taosd service, and then kill it
for proc in psutil.process_iter(): for proc in psutil.process_iter():
if proc.name() == 'taosd': if proc.name() == 'taosd':
print("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt") Logging.info("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt")
time.sleep(2.0) time.sleep(2.0)
proc.kill() proc.kill()
# print("Process: {}".format(proc.name())) # print("Process: {}".format(proc.name()))
@ -559,7 +593,8 @@ class ServiceManagerThread:
for i in range(0, 100): for i in range(0, 100):
time.sleep(1.0) time.sleep(1.0)
# self.procIpcBatch() # don't pump message during start up # self.procIpcBatch() # don't pump message during start up
print("_zz_", end="", flush=True) Progress.emit(Progress.SERVICE_START_NAP)
# print("_zz_", end="", flush=True)
if self._status.isRunning(): if self._status.isRunning():
Logging.info("[] TDengine service READY to process requests") Logging.info("[] TDengine service READY to process requests")
Logging.info("[] TAOS service started: {}".format(self)) Logging.info("[] TAOS service started: {}".format(self))
@ -595,12 +630,12 @@ class ServiceManagerThread:
def stop(self): def stop(self):
# can be called from both main thread or signal handler # can be called from both main thread or signal handler
print("Terminating TDengine service running as the sub process...") Logging.info("Terminating TDengine service running as the sub process...")
if self.getStatus().isStopped(): if self.getStatus().isStopped():
print("Service already stopped") Logging.info("Service already stopped")
return return
if self.getStatus().isStopping(): if self.getStatus().isStopping():
print("Service is already being stopped") Logging.info("Service is already being stopped")
return return
# Linux will send Control-C generated SIGINT to the TDengine process # Linux will send Control-C generated SIGINT to the TDengine process
# already, ref: # already, ref:
@ -616,10 +651,10 @@ class ServiceManagerThread:
if retCode == signal.SIGSEGV : # SGV if retCode == signal.SIGSEGV : # SGV
Logging.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)") Logging.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)")
except subprocess.TimeoutExpired as err: except subprocess.TimeoutExpired as err:
print("Time out waiting for TDengine service process to exit") Logging.info("Time out waiting for TDengine service process to exit")
else: else:
if self._tdeSubProcess.isRunning(): # still running, should now never happen if self._tdeSubProcess.isRunning(): # still running, should now never happen
print("FAILED to stop sub process, it is still running... pid = {}".format( Logging.error("FAILED to stop sub process, it is still running... pid = {}".format(
self._tdeSubProcess.getPid())) self._tdeSubProcess.getPid()))
else: else:
self._tdeSubProcess = None # not running any more self._tdeSubProcess = None # not running any more
@ -683,9 +718,9 @@ class ServiceManagerThread:
return # we are done with THIS BATCH return # we are done with THIS BATCH
else: # got line, printing out else: # got line, printing out
if forceOutput: if forceOutput:
Logging.info(line) Logging.info('[TAOSD] ' + line)
else: else:
Logging.debug(line) Logging.debug('[TAOSD] ' + line)
print(">", end="", flush=True) print(">", end="", flush=True)
_ProgressBars = ["--", "//", "||", "\\\\"] _ProgressBars = ["--", "//", "||", "\\\\"]
@ -728,11 +763,11 @@ class ServiceManagerThread:
# queue.put(line) # queue.put(line)
# meaning sub process must have died # meaning sub process must have died
Logging.info("\nEnd of stream detected for TDengine STDOUT: {}".format(self)) Logging.info("EOF for TDengine STDOUT: {}".format(self))
out.close() out.close()
def svcErrorReader(self, err: IO, queue): def svcErrorReader(self, err: IO, queue):
for line in iter(err.readline, b''): for line in iter(err.readline, b''):
print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line)) Logging.info("TDengine STDERR: {}".format(line))
Logging.info("\nEnd of stream detected for TDengine STDERR: {}".format(self)) Logging.info("EOF for TDengine STDERR: {}".format(self))
err.close() err.close()

2
tests/pytest/crash_gen_bootstrap.py
View File

@ -11,7 +11,7 @@
################################################################### ###################################################################
import sys import sys
from crash_gen.crash_gen import MainExec from crash_gen.crash_gen_main import MainExec
if __name__ == "__main__": if __name__ == "__main__":

16
tests/pytest/fulltest.sh
View File

@ -161,7 +161,7 @@ python3 ./test.py -f stream/metric_1.py
python3 ./test.py -f stream/new.py python3 ./test.py -f stream/new.py
python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream1.py
python3 ./test.py -f stream/stream2.py python3 ./test.py -f stream/stream2.py
python3 ./test.py -f stream/parser.py #python3 ./test.py -f stream/parser.py
python3 ./test.py -f stream/history.py python3 ./test.py -f stream/history.py
#alter table #alter table
@ -207,3 +207,17 @@ python3 test.py -f tools/taosdemo.py
python3 test.py -f subscribe/singlemeter.py python3 test.py -f subscribe/singlemeter.py
#python3 test.py -f subscribe/stability.py #python3 test.py -f subscribe/stability.py
python3 test.py -f subscribe/supertable.py python3 test.py -f subscribe/supertable.py
# update
python3 ./test.py -f update/allow_update.py
python3 ./test.py -f update/allow_update-0.py
python3 ./test.py -f update/append_commit_data.py
python3 ./test.py -f update/append_commit_last-0.py
python3 ./test.py -f update/append_commit_last.py
python3 ./test.py -f update/merge_commit_data.py
python3 ./test.py -f update/merge_commit_data-0.py
python3 ./test.py -f update/merge_commit_data2.py
python3 ./test.py -f update/merge_commit_data2_update0.py
python3 ./test.py -f update/merge_commit_last-0.py
python3 ./test.py -f update/merge_commit_last.py

11
tests/pytest/functions/function_percentile.py
View File

@ -132,6 +132,17 @@ class TDTestCase:
tdSql.query("select apercentile(col6, 100) from test") tdSql.query("select apercentile(col6, 100) from test")
print("apercentile result: %s" % tdSql.getData(0, 0)) print("apercentile result: %s" % tdSql.getData(0, 0))
tdSql.execute("create table meters (ts timestamp, voltage int) tags(loc nchar(20))")
tdSql.execute("create table t0 using meters tags('beijing')")
tdSql.execute("create table t1 using meters tags('shanghai')")
for i in range(self.rowNum):
tdSql.execute("insert into t0 values(%d, %d)" % (self.ts + i, i + 1))
tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i, i + 1))
tdSql.error("select percentile(voltage, 20) from meters")
tdSql.query("select apercentile(voltage, 20) from meters")
print("apercentile result: %s" % tdSql.getData(0, 0))
def stop(self): def stop(self):
tdSql.close() tdSql.close()
tdLog.success("%s successfully executed" % __file__) tdLog.success("%s successfully executed" % __file__)

4
tests/pytest/handle_crash_gen_val_log.sh
View File

@ -5,7 +5,9 @@ GREEN='\033[1;32m'
GREEN_DARK='\033[0;32m' GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m' GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m' NC='\033[0m'
nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null &
./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4
pidof taosd|xargs kill
grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log
for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'`

170
tests/pytest/update/allow_update-0.py Normal file
View File

@ -0,0 +1,170 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
print("==============step1")
tdSql.execute("create database udb update 0")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 1)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 1)
tdSql.checkData(6, 0, 3)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 1)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 3)
tdSql.checkData(7, 0, 1)
tdSql.checkData(8, 0, 3)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

266
tests/pytest/update/allow_update.py Normal file
View File

@ -0,0 +1,266 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self.numOfRecords = 10
self.ts = 1604295582000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
startTs = self.ts
tdSql.execute("create database udb update 1")
tdSql.execute("use udb")
tdSql.execute("create table t (ts timestamp, a int)")
print("==============step1")
tdSql.execute("insert into t values (%d, 1)" % (startTs))
tdSql.execute("insert into t values (%d, 1)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 1)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
print("==============step2")
tdSql.execute("insert into t values (%d, 2)" % (startTs))
tdSql.execute("insert into t values (%d, 2)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 2)" % (startTs + 3))
tdSql.query("select * from t")
tdSql.checkRows(3)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 2)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 2)
print("==============step3")
tdSql.execute("insert into t values (%d, 3)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 3)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 3)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 3)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 3)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 3)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 3)
print("==============step4")
tdSql.execute("insert into t values (%d, 4)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 4)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 4)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(7)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 2)
tdSql.checkData(4, 0, 4)
tdSql.checkData(5, 0, 2)
tdSql.checkData(6, 0, 4)
print("==============step5")
tdSql.execute("insert into t values (%d, 5)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 5)" % (startTs + 1))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 4)
tdSql.checkData(1, 0, 2)
tdSql.checkData(2, 0, 4)
tdSql.checkData(3, 0, 5)
tdSql.checkData(4, 0, 2)
tdSql.checkData(5, 0, 5)
tdSql.checkData(6, 0, 4)
tdSql.checkData(7, 0, 2)
tdSql.checkData(8, 0, 4)
print("==============step6")
tdSql.execute("insert into t values (%d, 6)" % (startTs - 4))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs - 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 1))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 2))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 3))
tdSql.execute("insert into t values (%d, 6)" % (startTs + 4))
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from t")
tdSql.checkRows(9)
tdSql.query("select a from t")
tdSql.checkData(0, 0, 6)
tdSql.checkData(1, 0, 6)
tdSql.checkData(2, 0, 6)
tdSql.checkData(3, 0, 6)
tdSql.checkData(4, 0, 6)
tdSql.checkData(5, 0, 6)
tdSql.checkData(6, 0, 6)
tdSql.checkData(7, 0, 6)
tdSql.checkData(8, 0, 6)
tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)")
print("==============step7")
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3))
tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, 1)
tdSql.checkData(1, 0, 1)
tdSql.checkData(2, 0, 1)
tdSql.checkData(0, 1, None)
tdSql.checkData(1, 1, None)
tdSql.checkData(2, 1, None)
tdSql.checkData(0, 2, 'c-3')
tdSql.checkData(1, 2, 'c+0')
tdSql.checkData(2, 2, 'c+3')
tdSql.checkData(0, 3, None)
tdSql.checkData(1, 3, None)
tdSql.checkData(2, 3, None)
print("==============step8")
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3))
tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3))
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
# restart taosd to commit, and check
self.restartTaosd();
tdSql.query("select * from subt")
tdSql.checkRows(3)
tdSql.query("select a,b,c,d from subt")
tdSql.checkData(0, 0, None)
tdSql.checkData(1, 0, None)
tdSql.checkData(2, 0, None)
tdSql.checkData(0, 1, 2.0)
tdSql.checkData(1, 1, 2.0)
tdSql.checkData(2, 1, 2.0)
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 2, None)
tdSql.checkData(2, 2, None)
tdSql.checkData(0, 3, 1)
tdSql.checkData(1, 3, 1)
tdSql.checkData(2, 3, 0)
tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))")
print("==============step9")
insertRows = 20000
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
for i in range(0, insertRows):
tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows))
tdSql.query("select * from ct")
tdSql.checkRows(insertRows)
tdSql.query("select a,b from ct limit 3")
tdSql.checkData(0, 0, insertRows+0)
tdSql.checkData(1, 0, insertRows+1)
tdSql.checkData(2, 0, insertRows+2)
tdSql.checkData(0, 1, insertRows+0)
tdSql.checkData(1, 1, insertRows+1)
tdSql.checkData(2, 1, insertRows+2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

84
tests/pytest/update/append_commit_data-0.py Normal file
View File

@ -0,0 +1,84 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("create table && insert data")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1604298064000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0+i))
print("==========step2")
print("restart to commit ")
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows)
for k in range(0,100):
tdLog.info("insert %d rows" % (insertRows))
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into db.t1 values(%d,1)' %
(t0+k*200+i)
)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows+200*k)
print("==========step2")
print("insert into another table ")
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 20000
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d, 1)' %
(t0+i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

84
tests/pytest/update/append_commit_data.py Normal file
View File

@ -0,0 +1,84 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("create table && insert data")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db update 1'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1604298064000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0+i))
print("==========step2")
print("restart to commit ")
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows)
for k in range(0,100):
tdLog.info("insert %d rows" % (insertRows))
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into db.t1 values(%d,1)' %
(t0+k*200+i)
)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from db.t1")
tdSql.checkRows(insertRows+200*k)
print("==========step2")
print("insert into another table ")
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 20000
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d, 1)' %
(t0+i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

90
tests/pytest/update/append_commit_last-0.py Normal file
View File

@ -0,0 +1,90 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1604298064000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use db")
def run(self):
tdSql.prepare()
print("==============step1")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(10):
tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(i + 1)
tdSql.query("select sum(a) from t1")
tdSql.checkData(0, 0, i + 1)
print("==============step2")
tdSql.execute("create table t2 (ts timestamp, a int)")
tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 1)
for i in range(1, 151):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(151)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 151)
print("==============step3")
tdSql.execute("create table t3 (ts timestamp, a int)")
tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 1)
for i in range(8):
for j in range(1, 11):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(81)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 81)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

85
tests/pytest/update/append_commit_last.py Normal file
View File

@ -0,0 +1,85 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1604298064000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
print("==============step1")
tdSql.execute("create database udb update 1")
tdSql.execute("use udb")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(10):
tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(i + 1)
print("==============step2")
tdSql.execute("create table t2 (ts timestamp, a int)")
tdSql.execute("insert into t2 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(1)
for i in range(1, 151):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(151)
print("==============step3")
tdSql.execute("create table t3 (ts timestamp, a int)")
tdSql.execute("insert into t3 values(%d, 1)" % self.ts)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(1)
for i in range(8):
for j in range(1, 11):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(81)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

351
tests/pytest/update/merge_commit_data-0.py Normal file
View File

@ -0,0 +1,351 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db days 30'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0 + i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
for k in range(0,10):
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into t1 values(%d,1)' %
(t0+i)
)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
print("==========step2")
print("PREPEND DATA ")
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
for i in range(-100,0):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
print("==========step3")
print("PREPEND MASSIVE DATA ")
ret = tdSql.execute('create table t3 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
for i in range(-6000,0):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
print("==========step4")
print("APPEND DATA")
ret = tdSql.execute('create table t4 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
print("==========step5")
print("APPEND DATA")
ret = tdSql.execute('create table t5 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
for i in range(0,6000):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
print("==========step6")
print("UPDATE BLOCK IN TWO STEP")
ret = tdSql.execute('create table t6 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t6 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
for i in range(0,200):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'200')
print("==========step7")
print("UPDATE LAST HALF AND INSERT LITTLE DATA")
ret = tdSql.execute('create table t7 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t7 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
for i in range(100,300):
ret = tdSql.execute(
'insert into t7 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'400')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'400')
print("==========step8")
print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
ret = tdSql.execute('create table t8 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t8 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
for i in range(6000):
ret = tdSql.execute(
'insert into t8 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'11800')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'11800')
print("==========step9")
print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
ret = tdSql.execute('create table t9 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t9 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
for i in range(-100,100):
ret = tdSql.execute(
'insert into t9 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'400')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'400')
print("==========step10")
print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
ret = tdSql.execute('create table t10 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t10 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
for i in range(-6000,100):
ret = tdSql.execute(
'insert into t10 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12200')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12200')
print("==========step11")
print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
ret = tdSql.execute('create table t11 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t11 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
for i in range(100):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
for i in range(200,6000):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11800')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11800')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

351
tests/pytest/update/merge_commit_data.py Normal file
View File

@ -0,0 +1,351 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def run(self):
print("==========step1")
print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY")
s = 'reset query cache'
tdSql.execute(s)
s = 'drop database if exists db'
tdSql.execute(s)
s = 'create database db update 1 days 30'
tdSql.execute(s)
s = 'use db'
tdSql.execute(s)
ret = tdSql.execute('create table t1 (ts timestamp, a int)')
insertRows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % (insertRows))
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t1 values (%d , 1)' %
(t0 + i))
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
for k in range(0,10):
for i in range (0,insertRows):
ret = tdSql.execute(
'insert into t1 values(%d,1)' %
(t0+i)
)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t1")
tdSql.checkRows(insertRows)
print("==========step2")
print("PREPEND DATA ")
ret = tdSql.execute('create table t2 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows)
for i in range(-100,0):
ret = tdSql.execute(
'insert into t2 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t2")
tdSql.checkRows(insertRows+100)
print("==========step3")
print("PREPEND MASSIVE DATA ")
ret = tdSql.execute('create table t3 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows)
for i in range(-6000,0):
ret = tdSql.execute(
'insert into t3 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t3")
tdSql.checkRows(insertRows+6000)
print("==========step4")
print("APPEND DATA")
ret = tdSql.execute('create table t4 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t4 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t4")
tdSql.checkRows(insertRows+100)
print("==========step5")
print("APPEND DATA")
ret = tdSql.execute('create table t5 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows)
for i in range(0,6000):
ret = tdSql.execute(
'insert into t5 values (%d , 1)' %
(t0+200+i))
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t5")
tdSql.checkRows(insertRows+6000)
print("==========step6")
print("UPDATE BLOCK IN TWO STEP")
ret = tdSql.execute('create table t6 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t6 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
for i in range(0,100):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'300')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'300')
for i in range(0,200):
ret = tdSql.execute(
'insert into t6 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'400')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t6")
tdSql.checkRows(insertRows)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0,0,'400')
print("==========step7")
print("UPDATE LAST HALF AND INSERT LITTLE DATA")
ret = tdSql.execute('create table t7 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t7 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(insertRows)
for i in range(100,300):
ret = tdSql.execute(
'insert into t7 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'500')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t7")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0,0,'500')
print("==========step8")
print("UPDATE LAST HALF AND INSERT MASSIVE DATA")
ret = tdSql.execute('create table t8 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t8 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(insertRows)
for i in range(6000):
ret = tdSql.execute(
'insert into t8 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'12000')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0,0,'12000')
print("==========step9")
print("UPDATE FIRST HALF AND PREPEND LITTLE DATA")
ret = tdSql.execute('create table t9 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t9 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(insertRows)
for i in range(-100,100):
ret = tdSql.execute(
'insert into t9 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'500')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t9")
tdSql.checkRows(300)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0,0,'500')
print("==========step10")
print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA")
ret = tdSql.execute('create table t10 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t10 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(insertRows)
for i in range(-6000,100):
ret = tdSql.execute(
'insert into t10 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12300')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t10")
tdSql.checkRows(6200)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0,0,'12300')
print("==========step11")
print("UPDATE FIRST HALF AND APPEND MASSIVE DATA")
ret = tdSql.execute('create table t11 (ts timestamp, a int)')
insertRows = 200
for i in range(0, insertRows):
ret = tdSql.execute(
'insert into t11 values (%d , 1)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(insertRows)
for i in range(100):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
for i in range(200,6000):
ret = tdSql.execute(
'insert into t11 values (%d , 2)' %
(t0+i))
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11900')
tdDnodes.stop(1)
tdDnodes.start(1)
tdSql.query("select * from t11")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t11")
tdSql.checkData(0,0,'11900')
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

352
tests/pytest/update/merge_commit_data2.py Normal file
View File

@ -0,0 +1,352 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
import time
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def restart_taosd(self,db):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use %s;" % db)
def date_to_timestamp_microseconds(self, date):
datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
return obj_stamp
def timestamp_microseconds_to_date(self, timestamp):
d = datetime.datetime.fromtimestamp(timestamp/1000)
str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
return str1
def run(self):
print("==========step1")
print("create table && insert data")
sql = 'reset query cache'
tdSql.execute(sql)
sql = 'drop database if exists db'
tdSql.execute(sql)
sql = 'create database db update 1 days 30;'
tdSql.execute(sql)
sql = 'use db;'
tdSql.execute(sql)
tdSql.execute('create table t1 (ts timestamp, a int)')
print("==================================1 start")
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
print("==========step2")
print("restart to commit ")
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
print("==========step3")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==========step4")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 2)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t1;')
# print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 2)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==================================2 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t2 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for k in range(10):
for i in range(10):
tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
print("==================================3 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t3 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(5200):
tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t3;')
for i in range(5200):
tdSql.checkData(i, 1, 2)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t3;')
for i in range(5200):
tdSql.checkData(i, 1, 2)
print("==================================4 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t4 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(100):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t4;')
for i in range(100):
tdSql.checkData(i, 1, 2)
for i in range(100, 200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t4;')
for i in range(100):
tdSql.checkData(i, 1, 2)
for i in range(100, 200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print("==================================5 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t5 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t5;')
for i in range(100):
tdSql.checkData(i, 1, 1)
for i in range(100, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5100):
tdSql.checkData(i, 1, 1)
for i in range(5100, 5200):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t5;')
for i in range(100):
tdSql.checkData(i, 1, 1)
for i in range(100, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5100):
tdSql.checkData(i, 1, 1)
for i in range(5100, 5200):
tdSql.checkData(i, 1, 2)
print("==================================6 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t6 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
print("==================================7 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t7 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(11000):
tdSql.checkData(i, 1, 2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

384
tests/pytest/update/merge_commit_data2_update0.py Normal file
View File

@ -0,0 +1,384 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
import time
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def restart_taosd(self,db):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use %s;" % db)
def date_to_timestamp_microseconds(self, date):
datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f")
obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0)
return obj_stamp
def timestamp_microseconds_to_date(self, timestamp):
d = datetime.datetime.fromtimestamp(timestamp/1000)
str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f")
return str1
def run(self):
print("==========step1")
print("create table && insert data")
sql = 'reset query cache'
tdSql.execute(sql)
sql = 'drop database if exists db'
tdSql.execute(sql)
sql = 'create database db update 0 days 30;'
tdSql.execute(sql)
sql = 'use db;'
tdSql.execute(sql)
tdSql.execute('create table t1 (ts timestamp, a int)')
print("==================================1 start")
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i))
print("==========step2")
print("restart to commit ")
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
print("==========step3")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t1;')
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==========step4")
print('insert data')
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 2)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t1;')
print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t1;')
# print(tdSql.queryResult)
for i in range(insert_rows):
tdSql.checkData(i, 1, 1)
for i in range(insert_rows, insert_rows*2):
tdSql.checkData(i, 1, 1)
print("==================================2 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t2 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for k in range(10):
for i in range(10):
tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
# print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t2;')
for i in range(insert_rows*2+100):
tdSql.checkData(i, 1, 1)
print("==================================3 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t3 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(5200):
tdSql.execute('insert into t3 values (%d , 2)' %(t0+i))
print("==========step2")
print('check query result before restart')
tdSql.query('select * from db.t3;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
# print(tdSql.queryResult)
print('restart to commit')
self.restart_taosd('db')
print('check query result after restart')
tdSql.query('select * from db.t3;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print("==================================4 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t4 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t4 values (%d , 2)' %(t0+i))
for i in range(100):
tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t4;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t4;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
#
print("==================================5 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t5 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(200, 5000):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i))
for i in range(100, 200):
tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000))
print('check query result before restart')
tdSql.query('select * from db.t5;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t5;')
for i in range(200):
tdSql.checkData(i, 1, 1)
for i in range(200, 5000):
tdSql.checkData(i, 1, 2)
for i in range(5000, 5200):
tdSql.checkData(i, 1, 1)
print("==================================6 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t6 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i))
print('restart to commit')
self.restart_taosd('db')
for i in range(insert_rows):
tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t6 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t6;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
print("==================================7 start")
print("==========step1")
print("create table && insert data")
tdSql.execute('create table t7 (ts timestamp, a int)')
insert_rows = 200
t0 = 1603152000000
tdLog.info("insert %d rows" % insert_rows)
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i))
for i in range(insert_rows):
tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000))
print('restart to commit')
self.restart_taosd('db')
for i in range(-1000, 10000):
tdSql.execute('insert into t7 values (%d , 2)' %(t0+i))
print('check query result before restart')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
print('check query result after restart')
self.restart_taosd('db')
tdSql.query('select * from db.t7;')
tdSql.checkRows(11000)
for i in range(1000):
tdSql.checkData(i, 1, 2)
for i in range(1000,1200):
tdSql.checkData(i, 1, 1)
for i in range(1200,6000):
tdSql.checkData(i, 1, 2)
for i in range(6000,6200):
tdSql.checkData(i, 1, 1)
for i in range(6200, 11000):
tdSql.checkData(i, 1, 2)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

309
tests/pytest/update/merge_commit_last-0.py Normal file
View File

@ -0,0 +1,309 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1603152000000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use db")
def run(self):
tdSql.prepare()
print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(5):
tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(1)
tdSql.checkData(0, 1, 0)
print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
tdSql.execute("create table t2 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 50)
for i in range(50):
tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t2")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 50)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t2")
tdSql.checkData(0, 0, 50)
print("==============step 3: UPDATE PART OF THE LAST BLOCK")
tdSql.execute("create table t3 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 50)
for i in range(25):
tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t3")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 50)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t3")
tdSql.checkData(0, 0, 50)
print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
tdSql.execute("create table t4 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t4")
tdSql.checkData(0, 0, 50)
for i in range(25):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
for i in range(50, 60):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t4")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t4")
tdSql.checkData(0, 0, 70)
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t4")
tdSql.checkData(0, 0, 70)
print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
tdSql.execute("create table t5 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 50)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
for i in range(25):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
for i in range(25, 50):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 70)
print("==============step 6: INSERT AHEAD A LOT OF DATA")
tdSql.execute("create table t6 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 0):
tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
tdSql.execute("create table t7 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 25):
tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2050)
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2050)
print("==============step 8: INSERT AFTER A LOT AND UPDATE")
tdSql.execute("create table t8 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 50)
for i in range(25, 6000):
tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11950)
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11950)
print("==============step 9: UPDATE ONLY MIDDLE")
tdSql.execute("create table t9 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
for i in range(20, 30):
tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
tdSql.execute("create table t10 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 50)
for i in range(-4000, 4000):
tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 15950)
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 15950)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

321
tests/pytest/update/merge_commit_last.py Normal file
View File

@ -0,0 +1,321 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import taos
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.ts = 1603152000000
def restartTaosd(self):
tdDnodes.stop(1)
tdDnodes.startWithoutSleep(1)
tdSql.execute("use udb")
def run(self):
tdSql.prepare()
tdSql.execute("create database udb update 1 days 30")
tdSql.execute("use udb")
print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY")
tdSql.execute("create table t1 (ts timestamp, a int)")
for i in range(5):
tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i))
self.restartTaosd()
tdSql.query("select * from t1")
tdSql.checkRows(1)
tdSql.checkData(0, 1, i)
print("==============step 2: UPDATE THE WHOLE LAST BLOCK")
tdSql.execute("create table t2 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(50):
tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t2")
for i in range(50):
tdSql.checkData(i, 1, 2)
self.restartTaosd()
tdSql.query("select * from t2")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 2)
print("==============step 3: UPDATE PART OF THE LAST BLOCK")
tdSql.execute("create table t3 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(25):
tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t3")
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
self.restartTaosd()
tdSql.query("select * from t3")
tdSql.checkRows(50)
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA")
tdSql.execute("create table t4 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(25):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
for i in range(50, 60):
tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t4")
tdSql.checkRows(60)
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
for i in range(50, 60):
tdSql.checkData(i, 1, 2)
self.restartTaosd()
tdSql.query("select * from t4")
tdSql.checkRows(60)
for i in range(25):
tdSql.checkData(i, 1, 2)
for i in range(25, 50):
tdSql.checkData(i, 1, 1)
for i in range(50, 60):
tdSql.checkData(i, 1, 2)
print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA")
tdSql.execute("create table t5 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(50)
for i in range(50):
tdSql.checkData(i, 1, 1)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
for i in range(25):
tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 95)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 95)
for i in range(-10, 0):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
for i in range(25, 50):
tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i))
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 155)
self.restartTaosd()
tdSql.query("select * from t5")
tdSql.checkRows(60)
tdSql.query("select sum(a) from t5")
tdSql.checkData(0, 0, 155)
print("==============step 6: INSERT AHEAD A LOT OF DATA")
tdSql.execute("create table t6 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 0):
tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
self.restartTaosd()
tdSql.query("select * from t6")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t6")
tdSql.checkData(0, 0, 2050)
print("==============step 7: INSERT AHEAD A LOT AND UPDATE")
tdSql.execute("create table t7 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 50)
for i in range(-1000, 25):
tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2075)
self.restartTaosd()
tdSql.query("select * from t7")
tdSql.checkRows(1050)
tdSql.query("select sum(a) from t7")
tdSql.checkData(0, 0, 2075)
print("==============step 8: INSERT AFTER A LOT AND UPDATE")
tdSql.execute("create table t8 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 50)
for i in range(25, 6000):
tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11975)
self.restartTaosd()
tdSql.query("select * from t8")
tdSql.checkRows(6000)
tdSql.query("select sum(a) from t8")
tdSql.checkData(0, 0, 11975)
print("==============step 9: UPDATE ONLY MIDDLE")
tdSql.execute("create table t9 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 50)
for i in range(20, 30):
tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 60)
self.restartTaosd()
tdSql.query("select * from t9")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t9")
tdSql.checkData(0, 0, 60)
print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK")
tdSql.execute("create table t10 (ts timestamp, a int)")
for i in range(50):
tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i))
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(50)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 50)
for i in range(-4000, 4000):
tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i))
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 16000)
self.restartTaosd()
tdSql.query("select * from t10")
tdSql.checkRows(8000)
tdSql.query("select sum(a) from t10")
tdSql.checkData(0, 0, 16000)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

2
tests/script/general/http/restful_full.sim
View File

@ -81,7 +81,7 @@ print =============== step2 - no db
#11 #11
system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql
print 11-> $system_content print 11-> $system_content
if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then
return -1 return -1
endi endi

114
tests/script/general/parser/join_multivnode.sim
View File

@ -134,66 +134,8 @@ sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbna
#1970-01-01 08:01:40.800 | 10 | 45.000000000 | 0 | true | false | 0 | #1970-01-01 08:01:40.800 | 10 | 45.000000000 | 0 | true | false | 0 |
#1970-01-01 08:01:40.790 | 10 | 945.000000000 | 90 | true | true | 0 | #1970-01-01 08:01:40.790 | 10 | 945.000000000 | 90 | true | true | 0 |
sql select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc limit 20 offset 19; sql_error select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc limit 20 offset 19;
if $rows != 20 then
return -1
endi
if $data00 != @70-01-01 08:01:40.800@ then
return -1
endi
if $data01 != 10 then
return -1
endi
if $data02 != 45.000000000 then
return -1
endi
if $data03 != 0 then
return -1
endi
if $data04 != 1 then
return -1
endi
if $data05 != 0 then
return -1
endi
if $data06 != 0 then
return -1
endi
if $data10 != @70-01-01 08:01:40.790@ then
return -1
endi
if $data11 != 10 then
return -1
endi
if $data12 != 945.000000000 then
return -1
endi
if $data13 != 90 then
return -1
endi
if $data14 != 1 then
return -1
endi
if $data15 != 1 then
return -1
endi
if $data16 != 0 then
return -1
endi
sql select count(join_mt0.c1), sum(join_mt0.c2)/count(*), avg(c2), first(join_mt0.c5), last(c7) from join_mt0 interval(10a) group by join_mt0.t1 order by join_mt0.ts desc; sql select count(join_mt0.c1), sum(join_mt0.c2)/count(*), avg(c2), first(join_mt0.c5), last(c7) from join_mt0 interval(10a) group by join_mt0.t1 order by join_mt0.ts desc;
if $rows != 100 then if $rows != 100 then
@ -261,59 +203,9 @@ if $data16 != 2 then
endi endi
# this function will cause shell crash # this function will cause shell crash
sql select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc; sql_error select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc;
if $rows != 100 then
return -1
endi
if $data00 != @70-01-01 08:01:40.990@ then sql_error select last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10m) group by join_mt0.t1 order by join_mt0.ts asc;
return -1
endi
if $data01 != 10 then
return -1
endi
if $data02 != 90 then
return -1
endi
if $data03 != 0 then
return -1
endi
if $data11 != 10 then
return -1
endi
if $data12 != 80 then
return -1
endi
if $data13 != 0 then
return -1
endi
sql select last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10m) group by join_mt0.t1 order by join_mt0.ts asc;
if $rows != 1 then
return -1
endi
if $data00 != @70-01-01 08:00:00.000@ then
return -1
endi
if $data01 != 1 then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data03 != 0 then
return -1
endi
sql_error select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); sql_error select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL);
sql select count(join_mt0.c1), first(join_mt0.c1)/count(*), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); sql select count(join_mt0.c1), first(join_mt0.c1)/count(*), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL);

12
tests/script/general/parser/stream_on_sys.sim
View File

@ -22,12 +22,12 @@ $i = 0
sql use $db sql use $db
sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) sliding(2s) sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s)
sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) sliding(2s) sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s)
sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) sliding(2s) sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s)
sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) sliding(2s) sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s)
sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) sliding(2s) sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s)
sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) sliding(2s) sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s)
sleep 120000 sleep 120000
sql select * from cpustrm sql select * from cpustrm
if $rows <= 0 then if $rows <= 0 then