xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into enh/TS-3737-3.0

This commit is contained in:
kailixu 2024-05-31 15:48:19 +08:00
parent 0ef202a56b 0988064f9c
commit 4ae8aeb909
87 changed files with 1784 additions and 829 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
deps/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/x64/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/mips/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/x86/dm_static/libdmodule.a vendored
View File

Binary file not shown.

33
include/common/tmsg.h
View File

@ -2366,20 +2366,31 @@ void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp* pRsp);
typedef struct {
int32_t dnodeId;
char* token;
} SMArbUpdateGroupReqMember;
} SMArbUpdateGroupMember;
typedef struct {
int32_t vgId;
int64_t dbUid;
SMArbUpdateGroupReqMember members[2];
int8_t isSync;
SMArbUpdateGroupReqMember assignedLeader;
int64_t version;
} SMArbUpdateGroupReq;
int32_t dnodeId;
char* token;
int8_t acked;
} SMArbUpdateGroupAssigned;
int32_t tSerializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq);
int32_t tDeserializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq);
void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq* pReq);
typedef struct {
int32_t vgId;
int64_t dbUid;
SMArbUpdateGroupMember members[2];
int8_t isSync;
int8_t assignedAcked;
SMArbUpdateGroupAssigned assignedLeader;
int64_t version;
} SMArbUpdateGroup;
typedef struct {
SArray* updateArray; // SMArbUpdateGroup
} SMArbUpdateGroupBatchReq;
int32_t tSerializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq);
int32_t tDeserializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq);
void tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq* pReq);
typedef struct {
char queryStrId[TSDB_QUERY_ID_LEN];

3
include/common/tmsgdef.h
View File

@ -388,7 +388,8 @@
TD_NEW_MSG_SEG(TDMT_MND_ARB_MSG) //9 << 8
TD_DEF_MSG_TYPE(TDMT_MND_ARB_HEARTBEAT_TIMER, "mnd-arb-hb-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_CHECK_SYNC_TIMER, "mnd-arb-check-sync-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) // no longer used
TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP_BATCH, "mnd-arb-update-group-batch", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_MAX_MSG, "mnd-arb-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_ARB_MSG)

5
include/dnode/mgmt/dnode.h
View File

@ -44,6 +44,11 @@ int32_t dmRun();
*/
void dmStop();
/**
* for tests
*/
bool dmReadyForTest();
#ifdef __cplusplus
}
#endif

2
include/dnode/mnode/mnode.h
View File

@ -101,6 +101,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
*/
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
int64_t mndGetRoleTimeMs(SMnode *pMnode);
/**
* @brief Process the rpc, sync request.
*

1
include/libs/nodes/plannodes.h
View File

@ -183,6 +183,7 @@ typedef struct SProjectLogicNode {
char stmtName[TSDB_TABLE_NAME_LEN];
bool ignoreGroupId;
bool inputIgnoreGroup;
bool isSetOpProj;
} SProjectLogicNode;
typedef struct SIndefRowsFuncLogicNode {

2
include/libs/nodes/querynodes.h
View File

@ -415,6 +415,7 @@ typedef struct SSelectStmt {
int32_t returnRows; // EFuncReturnRows
ETimeLineMode timeLineCurMode;
ETimeLineMode timeLineResMode;
bool timeLineFromOrderBy;
bool isEmptyResult;
bool isSubquery;
bool hasAggFuncs;
@ -453,6 +454,7 @@ typedef struct SSetOperator {
char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision;
ETimeLineMode timeLineResMode;
bool timeLineFromOrderBy;
bool joinContains;
} SSetOperator;

11
include/util/taoserror.h
View File

@ -327,7 +327,6 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) //
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
#define TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x039C)
// mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)
@ -612,6 +611,16 @@ int32_t* taosGetErrno();
#define TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0821)
#define TSDB_CODE_GRANT_DUPLICATED_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x0822)
#define TSDB_CODE_GRANT_VIEW_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0823)
#define TSDB_CODE_GRANT_BASIC_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0824)
#define TSDB_CODE_GRANT_STREAM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0825)
#define TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0826)
#define TSDB_CODE_GRANT_VIEW_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0827)
#define TSDB_CODE_GRANT_AUDIT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0828)
#define TSDB_CODE_GRANT_CSV_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0829)
#define TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082A)
#define TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082B)
#define TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082C)
#define TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082D)
// sync
// #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x

1
source/client/src/clientHb.c
View File

@ -92,6 +92,7 @@ static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *bat
}
if (!pRsp) {
releaseTscObj(pReq->connKey.tscRid);
taosHashCancelIterate(hbMgr->activeInfo, pReq);
break;
}
}

1
source/common/src/systable.c
View File

@ -76,6 +76,7 @@ static const SSysDbTableSchema arbGroupsSchema[] = {
{.name = "is_sync", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "assigned_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "assigned_token", .bytes = TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "assigned_acked", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
};
static const SSysDbTableSchema clusterSchema[] = {

2
source/common/src/tdataformat.c
View File

@ -2907,7 +2907,7 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt
}
} else {
if (varDataTLen(data + offset) > bytes) {
uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset),
uError("var data length invalid, varDataTLen(data + offset):%d >= bytes:%d", (int)varDataTLen(data + offset),
bytes);
code = TSDB_CODE_PAR_VALUE_TOO_LONG;
goto _exit;

93
source/common/src/tmsg.c
View File

@ -6441,21 +6441,33 @@ void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp *pRsp) {
taosMemoryFreeClear(pRsp->memberToken);
}
int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) {
int32_t tSerializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
if (tEncodeI32(&encoder, pReq->members[i].dnodeId) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->members[i].token) < 0) return -1;
int32_t sz = taosArrayGetSize(pReq->updateArray);
if (tEncodeI32(&encoder, sz) < 0) return -1;
for (int32_t i = 0; i < sz; i++) {
SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i);
if (tEncodeI32(&encoder, pGroup->vgId) < 0) return -1;
if (tEncodeI64(&encoder, pGroup->dbUid) < 0) return -1;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
if (tEncodeI32(&encoder, pGroup->members[i].dnodeId) < 0) return -1;
if (tEncodeCStr(&encoder, pGroup->members[i].token) < 0) return -1;
}
if (tEncodeI8(&encoder, pGroup->isSync) < 0) return -1;
if (tEncodeI32(&encoder, pGroup->assignedLeader.dnodeId) < 0) return -1;
if (tEncodeCStr(&encoder, pGroup->assignedLeader.token) < 0) return -1;
if (tEncodeI64(&encoder, pGroup->version) < 0) return -1;
}
for (int32_t i = 0; i < sz; i++) {
SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i);
if (tEncodeI8(&encoder, pGroup->assignedLeader.acked) < 0) return -1;
}
if (tEncodeI8(&encoder, pReq->isSync) < 0) return -1;
if (tEncodeI32(&encoder, pReq->assignedLeader.dnodeId) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->assignedLeader.token) < 0) return -1;
if (tEncodeI64(&encoder, pReq->version) < 0) return -1;
tEndEncode(&encoder);
@ -6464,23 +6476,44 @@ int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGrou
return tlen;
}
int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) {
int32_t tDeserializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
if (tDecodeI32(&decoder, &pReq->members[i].dnodeId) < 0) return -1;
pReq->members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->members[i].token) < 0) return -1;
int32_t sz = 0;
if (tDecodeI32(&decoder, &sz) < 0) return -1;
SArray *updateArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup));
if (!updateArray) return -1;
for (int32_t i = 0; i < sz; i++) {
SMArbUpdateGroup group = {0};
if (tDecodeI32(&decoder, &group.vgId) < 0) return -1;
if (tDecodeI64(&decoder, &group.dbUid) < 0) return -1;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
if (tDecodeI32(&decoder, &group.members[i].dnodeId) < 0) return -1;
group.members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, group.members[i].token) < 0) return -1;
}
if (tDecodeI8(&decoder, &group.isSync) < 0) return -1;
if (tDecodeI32(&decoder, &group.assignedLeader.dnodeId) < 0) return -1;
group.assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, group.assignedLeader.token) < 0) return -1;
if (tDecodeI64(&decoder, &group.version) < 0) return -1;
group.assignedLeader.acked = false;
taosArrayPush(updateArray, &group);
}
if (tDecodeI8(&decoder, &pReq->isSync) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->assignedLeader.dnodeId) < 0) return -1;
pReq->assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
if (tDecodeCStrTo(&decoder, pReq->assignedLeader.token) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->version) < 0) return -1;
if (!tDecodeIsEnd(&decoder)) {
for (int32_t i = 0; i < sz; i++) {
SMArbUpdateGroup *pGroup = taosArrayGet(updateArray, i);
if (tDecodeI8(&decoder, &pGroup->assignedLeader.acked) < 0) return -1;
}
}
pReq->updateArray = updateArray;
tEndDecode(&decoder);
@ -6488,14 +6521,20 @@ int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGr
return 0;
}
void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq *pReq) {
if (NULL == pReq) {
void tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq *pReq) {
if (NULL == pReq || NULL == pReq->updateArray) {
return;
}
for (int i = 0; i < 2; i++) {
taosMemoryFreeClear(pReq->members[i].token);
int32_t sz = taosArrayGetSize(pReq->updateArray);
for (int32_t i = 0; i < sz; i++) {
SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i);
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
taosMemoryFreeClear(pGroup->members[i].token);
}
taosMemoryFreeClear(pGroup->assignedLeader.token);
}
taosMemoryFreeClear(pReq->assignedLeader.token);
taosArrayDestroy(pReq->updateArray);
}
// int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {

4
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -415,3 +415,7 @@ void dmReportStartup(const char *pName, const char *pDesc) {
}
int64_t dmGetClusterId() { return globalDnode.data.clusterId; }
bool dmReadyForTest() {
return dmInstance()->data.dnodeVer > 0;
}

4
source/dnode/mgmt/test/sut/inc/server.h
View File

@ -20,10 +20,10 @@ class TestServer {
public:
bool Start();
void Stop();
bool runnning;
bool running;
private:
TdThread threadId;
};
#endif /* _TD_TEST_SERVER_H_ */
#endif /* _TD_TEST_SERVER_H_ */

11
source/dnode/mgmt/test/sut/src/server.cpp
View File

@ -17,13 +17,11 @@
void* serverLoop(void* param) {
TestServer* server = (TestServer*)param;
server->runnning = false;
if (dmInit() != 0) {
return NULL;
}
server->runnning = true;
if (dmRun() != 0) {
return NULL;
}
@ -33,13 +31,18 @@ void* serverLoop(void* param) {
}
bool TestServer::Start() {
tstrncpy(tsVersionName, "trial", strlen("trial"));
running = false;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
taosThreadCreate(&threadId, &thAttr, serverLoop, this);
taosThreadAttrDestroy(&thAttr);
taosMsleep(2100);
return runnning;
while (!dmReadyForTest()) {
taosMsleep(500);
}
running = true;
return running;
}
void TestServer::Stop() {

3
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -180,7 +180,7 @@ typedef struct {
tmsg_t originRpcType;
char dbname[TSDB_TABLE_FNAME_LEN];
char stbname[TSDB_TABLE_FNAME_LEN];
int32_t arbGroupId;
SHashObj* arbGroupIds;
int32_t startFunc;
int32_t stopFunc;
int32_t paramLen;
@ -255,6 +255,7 @@ typedef struct {
typedef struct {
int32_t dnodeId;
char token[TSDB_ARB_TOKEN_SIZE];
int8_t acked;
} SArbAssignedLeader;
typedef struct {

2
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -78,7 +78,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen);
void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname);
void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId);
void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId);
void mndTransSetSerial(STrans *pTrans);
void mndTransSetParallel(STrans *pTrans);
void mndTransSetChangeless(STrans *pTrans);

302
source/dnode/mnode/impl/src/mndArbGroup.c
View File

@ -25,7 +25,7 @@
#include "mndVgroup.h"
#define ARBGROUP_VER_NUMBER 1
#define ARBGROUP_RESERVE_SIZE 64
#define ARBGROUP_RESERVE_SIZE 63
static SHashObj *arbUpdateHash = NULL;
@ -39,10 +39,11 @@ static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup);
static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew);
static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup);
static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray);
static int32_t mndProcessArbHbTimer(SRpcMsg *pReq);
static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq);
static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq);
static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq);
static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp);
static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp);
static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp);
@ -68,7 +69,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_ARB_HEARTBEAT_TIMER, mndProcessArbHbTimer);
mndSetMsgHandle(pMnode, TDMT_MND_ARB_CHECK_SYNC_TIMER, mndProcessArbCheckSyncTimer);
mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP, mndProcessArbUpdateGroupReq);
mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP_BATCH, mndProcessArbUpdateGroupBatchReq);
mndSetMsgHandle(pMnode, TDMT_VND_ARB_HEARTBEAT_RSP, mndProcessArbHbRsp);
mndSetMsgHandle(pMnode, TDMT_VND_ARB_CHECK_SYNC_RSP, mndProcessArbCheckSyncRsp);
mndSetMsgHandle(pMnode, TDMT_SYNC_SET_ASSIGNED_LEADER_RSP, mndProcessArbSetAssignedLeaderRsp);
@ -81,9 +82,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupArbGroup(SMnode *pMnode) {
taosHashCleanup(arbUpdateHash);
}
void mndCleanupArbGroup(SMnode *pMnode) { taosHashCleanup(arbUpdateHash); }
SArbGroup *mndAcquireArbGroup(SMnode *pMnode, int32_t vgId) {
SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId);
@ -130,6 +129,7 @@ SSdbRaw *mndArbGroupActionEncode(SArbGroup *pGroup) {
SDB_SET_INT32(pRaw, dataPos, pLeader->dnodeId, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER)
SDB_SET_INT64(pRaw, dataPos, pGroup->version, _OVER)
SDB_SET_INT8(pRaw, dataPos, pLeader->acked, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, ARBGROUP_RESERVE_SIZE, _OVER)
@ -183,6 +183,7 @@ SSdbRow *mndArbGroupActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pLeader->dnodeId, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pGroup->version, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pLeader->acked, _OVER)
pGroup->mutexInited = false;
@ -236,6 +237,7 @@ static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *p
pOld->isSync = pNew->isSync;
pOld->assignedLeader.dnodeId = pNew->assignedLeader.dnodeId;
memcpy(pOld->assignedLeader.token, pNew->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
pOld->assignedLeader.acked = pNew->assignedLeader.acked;
pOld->version++;
_OVER:
@ -541,6 +543,16 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
return -1;
}
int64_t roleTimeMs = mndGetRoleTimeMs(pMnode);
int64_t nowMs = taosGetTimestampMs();
if (nowMs - roleTimeMs < tsArbHeartBeatIntervalSec * 1000 * 2) {
mInfo("arb skip to check sync since mnd had just switch over, roleTime:%" PRId64 " now:%" PRId64, roleTimeMs,
nowMs);
return 0;
}
SArray *pUpdateArray = taosArrayInit(16, sizeof(SArbGroup));
while (1) {
pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup);
if (pIter == NULL) break;
@ -550,15 +562,14 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
taosThreadMutexUnlock(&pArbGroup->mutex);
int32_t vgId = arbGroupDup.vgId;
int64_t nowMs = taosGetTimestampMs();
bool member0IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 0, nowMs);
bool member1IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 1, nowMs);
SArbAssignedLeader *pAssignedLeader = &arbGroupDup.assignedLeader;
int32_t currentAssignedDnodeId = pAssignedLeader->dnodeId;
// 1. has assigned && is sync => send req
if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true) {
// 1. has assigned && is sync && no response => send req
if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true && pAssignedLeader->acked == false) {
(void)mndSendArbSetAssignedLeaderReq(pMnode, currentAssignedDnodeId, vgId, arbToken, term,
pAssignedLeader->token);
mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, currentAssignedDnodeId);
@ -612,40 +623,27 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
SArbGroup newGroup = {0};
mndArbGroupDupObj(&arbGroupDup, &newGroup);
mndArbGroupSetAssignedLeader(&newGroup, candidateIndex);
if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) {
mError("vgId:%d, arb failed to pullup set assigned leader to dnodeId:%d, since %s", vgId, pMember->info.dnodeId,
terrstr());
sdbRelease(pSdb, pArbGroup);
return -1;
}
mInfo("vgId:%d, arb pull up set assigned leader to dnodeId:%d", vgId, pMember->info.dnodeId);
taosArrayPush(pUpdateArray, &newGroup);
sdbRelease(pSdb, pArbGroup);
}
(void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray);
taosArrayDestroy(pUpdateArray);
return 0;
}
static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup) {
SMArbUpdateGroupReq req = {0};
req.vgId = pNewGroup->vgId;
req.dbUid = pNewGroup->dbUid;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
req.members[i].dnodeId = pNewGroup->members[i].info.dnodeId;
req.members[i].token = pNewGroup->members[i].state.token;
}
req.isSync = pNewGroup->isSync;
req.assignedLeader.dnodeId = pNewGroup->assignedLeader.dnodeId;
req.assignedLeader.token = pNewGroup->assignedLeader.token;
req.version = pNewGroup->version;
static void *mndBuildArbUpdateGroupBatchReq(int32_t *pContLen, SArray *updateArray) {
SMArbUpdateGroupBatchReq req = {0};
req.updateArray = updateArray;
int32_t contLen = tSerializeSMArbUpdateGroupReq(NULL, 0, &req);
int32_t contLen = tSerializeSMArbUpdateGroupBatchReq(NULL, 0, &req);
if (contLen <= 0) return NULL;
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) return NULL;
if (tSerializeSMArbUpdateGroupReq(pHead, contLen, &req) <= 0) {
if (tSerializeSMArbUpdateGroupBatchReq(pHead, contLen, &req) <= 0) {
rpcFreeCont(pHead);
return NULL;
}
@ -653,60 +651,174 @@ static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup)
return pHead;
}
static void mndInitArbUpdateGroup(SArbGroup *pGroup, SMArbUpdateGroup *outGroup) {
outGroup->vgId = pGroup->vgId;
outGroup->dbUid = pGroup->dbUid;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
outGroup->members[i].dnodeId = pGroup->members[i].info.dnodeId;
outGroup->members[i].token = pGroup->members[i].state.token; // just copy the pointer
}
outGroup->isSync = pGroup->isSync;
outGroup->assignedLeader.dnodeId = pGroup->assignedLeader.dnodeId;
outGroup->assignedLeader.token = pGroup->assignedLeader.token; // just copy the pointer
outGroup->assignedLeader.acked = pGroup->assignedLeader.acked;
outGroup->version = pGroup->version;
}
static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup) {
if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) {
mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId);
return 0;
}
int32_t contLen = 0;
void *pHead = mndBuildArbUpdateGroupReq(&contLen, pNewGroup);
if (!pHead) {
mError("vgId:%d, failed to build arb-update-group request", pNewGroup->vgId);
return -1;
}
SRpcMsg rpcMsg = {.msgType = TDMT_MND_ARB_UPDATE_GROUP, .pCont = pHead, .contLen = contLen, .info.noResp = true};
int32_t ret = -1;
int32_t ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
if (ret == 0) {
taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
SMArbUpdateGroup newGroup = {0};
mndInitArbUpdateGroup(pNewGroup, &newGroup);
SArray *pArray = taosArrayInit(1, sizeof(SMArbUpdateGroup));
taosArrayPush(pArray, &newGroup);
int32_t contLen = 0;
void *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray);
if (!pHead) {
mError("failed to build arb-update-group request");
goto _OVER;
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true};
ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
if (ret != 0) goto _OVER;
taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
_OVER:
taosArrayDestroy(pArray);
return ret;
}
static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq) {
int ret = 0;
static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray) {
int32_t ret = -1;
SMArbUpdateGroupReq req = {0};
tDeserializeSMArbUpdateGroupReq(pReq->pCont, pReq->contLen, &req);
size_t sz = taosArrayGetSize(newGroupArray);
SArray *pArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup));
for (size_t i = 0; i < sz; i++) {
SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i);
if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) {
mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId);
continue;
}
SArbGroup newGroup = {0};
newGroup.vgId = req.vgId;
newGroup.dbUid = req.dbUid;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
newGroup.members[i].info.dnodeId = req.members[i].dnodeId;
memcpy(newGroup.members[i].state.token, req.members[i].token, TSDB_ARB_TOKEN_SIZE);
SMArbUpdateGroup newGroup = {0};
mndInitArbUpdateGroup(pNewGroup, &newGroup);
taosArrayPush(pArray, &newGroup);
taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
}
newGroup.isSync = req.isSync;
newGroup.assignedLeader.dnodeId = req.assignedLeader.dnodeId;
memcpy(newGroup.assignedLeader.token, req.assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
newGroup.version = req.version;
SMnode *pMnode = pReq->info.node;
SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId);
if (!pOldGroup) {
mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId);
return 0;
}
sdbRelease(pMnode->pSdb, pOldGroup);
if (mndArbGroupUpdateTrans(pMnode, &newGroup) != 0) {
mError("vgId:%d, arb failed to update arbgroup, since %s", newGroup.vgId, terrstr());
ret = -1;
if (taosArrayGetSize(pArray) == 0) {
ret = 0;
goto _OVER;
}
tFreeSMArbUpdateGroupReq(&req);
int32_t contLen = 0;
void *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray);
if (!pHead) {
mError("failed to build arb-update-group request");
goto _OVER;
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true};
ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
_OVER:
taosArrayDestroy(pArray);
if (ret != 0) {
for (size_t i = 0; i < sz; i++) {
SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i);
taosHashRemove(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId));
}
}
return ret;
}
static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
int ret = -1;
size_t sz = 0;
SMArbUpdateGroupBatchReq req = {0};
if (tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req) != 0) {
mError("arb failed to decode arb-update-group request");
return -1;
}
SMnode *pMnode = pReq->info.node;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup");
if (pTrans == NULL) {
mError("failed to update arbgroup in create trans, since %s", terrstr());
goto _OVER;
}
sz = taosArrayGetSize(req.updateArray);
for (size_t i = 0; i < sz; i++) {
SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i);
SArbGroup newGroup = {0};
newGroup.vgId = pUpdateGroup->vgId;
newGroup.dbUid = pUpdateGroup->dbUid;
for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
newGroup.members[i].info.dnodeId = pUpdateGroup->members[i].dnodeId;
memcpy(newGroup.members[i].state.token, pUpdateGroup->members[i].token, TSDB_ARB_TOKEN_SIZE);
}
newGroup.isSync = pUpdateGroup->isSync;
newGroup.assignedLeader.dnodeId = pUpdateGroup->assignedLeader.dnodeId;
memcpy(newGroup.assignedLeader.token, pUpdateGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
newGroup.assignedLeader.acked = pUpdateGroup->assignedLeader.acked;
newGroup.version = pUpdateGroup->version;
SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId);
if (!pOldGroup) {
mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId);
taosHashRemove(arbUpdateHash, &newGroup.vgId, sizeof(int32_t));
continue;
}
mndTransAddArbGroupId(pTrans, newGroup.vgId);
if (mndSetCreateArbGroupCommitLogs(pTrans, &newGroup) != 0) {
mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id,
terrstr());
goto _OVER;
}
mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]",
pTrans->id, newGroup.vgId, newGroup.members[0].info.dnodeId, newGroup.members[0].state.token,
newGroup.members[1].info.dnodeId, newGroup.members[1].state.token, newGroup.isSync,
newGroup.assignedLeader.dnodeId, newGroup.assignedLeader.token, newGroup.assignedLeader.acked);
sdbRelease(pMnode->pSdb, pOldGroup);
}
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
ret = 0;
_OVER:
if (ret != 0) {
// failed to update arbgroup
for (size_t i = 0; i < sz; i++) {
SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i);
taosHashRemove(arbUpdateHash, &pUpdateGroup->vgId, sizeof(int32_t));
}
}
mndTransDrop(pTrans);
tFreeSMArbUpdateGroupBatchReq(&req);
return ret;
}
@ -719,11 +831,13 @@ static void mndArbGroupSetAssignedLeader(SArbGroup *pGroup, int32_t index) {
pGroup->assignedLeader.dnodeId = pMember->info.dnodeId;
strncpy(pGroup->assignedLeader.token, pMember->state.token, TSDB_ARB_TOKEN_SIZE);
pGroup->assignedLeader.acked = false;
}
static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup) {
pGroup->assignedLeader.dnodeId = 0;
memset(pGroup->assignedLeader.token, 0, TSDB_ARB_TOKEN_SIZE);
pGroup->assignedLeader.acked = false;
}
static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
@ -734,12 +848,12 @@ static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
goto _OVER;
}
mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s]",
mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]",
pTrans->id, pNew->vgId, pNew->members[0].info.dnodeId, pNew->members[0].state.token,
pNew->members[1].info.dnodeId, pNew->members[1].state.token, pNew->isSync, pNew->assignedLeader.dnodeId,
pNew->assignedLeader.token);
pNew->assignedLeader.token, pNew->assignedLeader.acked);
mndTransSetArbGroupId(pTrans, pNew->vgId);
mndTransAddArbGroupId(pTrans, pNew->vgId);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
ret = -1;
goto _OVER;
@ -816,10 +930,10 @@ _OVER:
}
static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *memberArray) {
int ret = 0;
int64_t nowMs = taosGetTimestampMs();
size_t size = taosArrayGetSize(memberArray);
SArray *pUpdateArray = taosArrayInit(size, sizeof(SArbGroup));
size_t size = taosArrayGetSize(memberArray);
for (size_t i = 0; i < size; i++) {
SVArbHbRspMember *pRspMember = taosArrayGet(memberArray, i);
@ -832,17 +946,16 @@ static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *me
bool updateToken = mndUpdateArbGroupByHeartBeat(pGroup, pRspMember, nowMs, dnodeId, &newGroup);
if (updateToken) {
ret = mndPullupArbUpdateGroup(pMnode, &newGroup);
if (ret != 0) {
mInfo("failed to pullup update arb token, vgId:%d, since %s", pRspMember->vgId, terrstr());
}
taosArrayPush(pUpdateArray, &newGroup);
}
sdbRelease(pMnode->pSdb, pGroup);
if (ret != 0) break;
}
return ret;
(void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray);
taosArrayDestroy(pUpdateArray);
return 0;
}
bool mndUpdateArbGroupByCheckSync(SArbGroup *pGroup, int32_t vgId, char *member0Token, char *member1Token,
@ -900,6 +1013,11 @@ static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token
}
static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) {
if (pRsp->contLen == 0) {
mDebug("arb hb-rsp contLen is 0");
return 0;
}
int32_t ret = -1;
SMnode *pMnode = pRsp->info.node;
@ -914,6 +1032,7 @@ static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) {
SVArbHeartBeatRsp arbHbRsp = {0};
if (tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp) != 0) {
mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
@ -934,6 +1053,11 @@ _OVER:
}
static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
if (pRsp->contLen == 0) {
mDebug("arb check-sync-rsp contLen is 0");
return 0;
}
int32_t ret = -1;
SMnode *pMnode = pRsp->info.node;
@ -948,7 +1072,7 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
SVArbCheckSyncRsp syncRsp = {0};
if (tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp) != 0) {
mInfo("arb sync check failed, since:%s", tstrerror(pRsp->code));
mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code));
if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) {
terrno = TSDB_CODE_SUCCESS;
return 0;
@ -993,11 +1117,12 @@ bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char
goto _OVER;
}
if (pGroup->isSync) {
if (pGroup->assignedLeader.acked == false) {
mndArbGroupDupObj(pGroup, pNewGroup);
pNewGroup->isSync = false;
pNewGroup->assignedLeader.acked = true;
mInfo("vgId:%d, arb isSync is setting to false", vgId);
mInfo("vgId:%d, arb received assigned ack", vgId);
updateAssigned = true;
goto _OVER;
}
@ -1008,6 +1133,11 @@ _OVER:
}
static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
if (pRsp->contLen == 0) {
mDebug("arb set-assigned-rsp contLen is 0");
return 0;
}
int32_t ret = -1;
SMnode *pMnode = pRsp->info.node;
@ -1022,8 +1152,8 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
SVArbSetAssignedLeaderRsp setAssignedRsp = {0};
if (tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp) != 0) {
mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG;
mInfo("arb set assigned failed, des failed since:%s", tstrerror(pRsp->code));
return -1;
}
@ -1102,12 +1232,18 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
STR_WITH_MAXSIZE_TO_VARSTR(token, pGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)token, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pGroup->assignedLeader.acked, false);
} else {
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetNULL(pColInfo, numOfRows);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetNULL(pColInfo, numOfRows);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetNULL(pColInfo, numOfRows);
}
taosThreadMutexUnlock(&pGroup->mutex);

5
source/dnode/mnode/impl/src/mndDb.c
View File

@ -821,8 +821,7 @@ static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) {
#ifdef TD_ENTERPRISE
if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit;
if (grantCheck(TSDB_GRANT_DB_ENCRYPTION) != 0) {
code = TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED;
if ((code = grantCheck(TSDB_GRANT_DB_ENCRYPTION)) != 0) {
goto _exit;
}
if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) {
@ -1226,7 +1225,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
if (terrno != 0) code = terrno;
mError("db:%s, failed to alter since %s", alterReq.db, terrstr());
mError("db:%s, failed to alter since %s", alterReq.db, tstrerror(code));
}
mndReleaseDb(pMnode, pDb);

31
source/dnode/mnode/impl/src/mndMain.c
View File

@ -334,6 +334,8 @@ static int32_t minCronTime() {
min = TMIN(min, tsStreamCheckpointInterval);
min = TMIN(min, 6); // checkpointRemain
min = TMIN(min, tsStreamNodeCheckInterval);
min = TMIN(min, tsArbHeartBeatIntervalSec);
min = TMIN(min, tsArbCheckSyncIntervalSec);
int64_t telemInt = TMIN(60, (tsTelemInterval - 1));
min = TMIN(min, telemInt);
@ -390,6 +392,18 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
if (sec % tsUptimeInterval == 0) {
mndIncreaseUpTime(pMnode);
}
if (sec % (tsArbHeartBeatIntervalSec) == 0) {
if (mndPullupArbHeartbeat(pMnode) != 0) {
mError("failed to pullup arb heartbeat, since:%s", terrstr());
}
}
if (sec % (tsArbCheckSyncIntervalSec) == 0) {
if (mndPullupArbCheckSync(pMnode) != 0) {
mError("failed to pullup arb check sync, since:%s", terrstr());
}
}
}
void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) {
if (sec % (tsStatusInterval * 5) == 0) {
@ -421,18 +435,6 @@ static void *mndThreadFp(void *param) {
continue;
}
mndDoTimerPullupTask(pMnode, sec);
if (sec % (tsArbHeartBeatIntervalSec) == 0) {
if (mndPullupArbHeartbeat(pMnode) != 0) {
mError("failed to pullup arb heartbeat, since:%s", terrstr());
}
}
if (sec % (tsArbCheckSyncIntervalSec) == 0) {
if (mndPullupArbCheckSync(pMnode) != 0) {
mError("failed to pullup arb check sync, since:%s", terrstr());
}
}
}
return NULL;
@ -1076,6 +1078,11 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
return 0;
}
int64_t mndGetRoleTimeMs(SMnode *pMnode) {
SSyncState state = syncGetState(pMnode->syncMgmt.sync);
return state.roleTimeMs;
}
void mndSetRestored(SMnode *pMnode, bool restored) {
if (restored) {
taosThreadRwlockWrlock(&pMnode->lock);

7
source/dnode/mnode/impl/src/mndStream.c
View File

@ -1639,11 +1639,11 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
// info
if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
const char *sinkStr = "%.2fMiB";
sprintf(buf, sinkStr, pe->sinkDataSize);
snprintf(buf, tListLen(buf), sinkStr, pe->sinkDataSize);
} else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
// offset info
const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]";
sprintf(buf, offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer);
snprintf(buf, tListLen(buf), offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer);
}
STR_TO_VARSTR(vbuf, buf);
@ -1847,8 +1847,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL;
if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) {
terrno = TSDB_CODE_GRANT_EXPIRED;
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
return -1;
}

2
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -225,7 +225,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SArray *pFailedTasks = NULL;
SArray *pOrphanTasks = NULL;
if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) {
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
if (suspendAllStreams(pMnode, &pReq->info) < 0) {
return -1;
}

47
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -26,7 +26,7 @@
#define TRANS_VER1_NUMBER 1
#define TRANS_VER2_NUMBER 2
#define TRANS_ARRAY_SIZE 8
#define TRANS_RESERVE_SIZE 48
#define TRANS_RESERVE_SIZE 44
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld);
@ -196,10 +196,21 @@ SSdbRaw *mndTransEncode(STrans *pTrans) {
}
SDB_SET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER)
int32_t arbGroupNum = taosHashGetSize(pTrans->arbGroupIds);
SDB_SET_INT32(pRaw, dataPos, arbGroupNum, _OVER)
void *pIter = NULL;
pIter = taosHashIterate(pTrans->arbGroupIds, NULL);
while (pIter) {
int32_t arbGroupId = *(int32_t *)pIter;
SDB_SET_INT32(pRaw, dataPos, arbGroupId, _OVER)
pIter = taosHashIterate(pTrans->arbGroupIds, pIter);
}
SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
terrno = 0;
terrno = 0;
_OVER:
if (terrno != 0) {
@ -279,6 +290,7 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) {
int32_t undoActionNum = 0;
int32_t commitActionNum = 0;
int32_t dataPos = 0;
int32_t arbgroupIdNum = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -350,6 +362,16 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) {
}
SDB_GET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER);
pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
SDB_GET_INT32(pRaw, dataPos, &arbgroupIdNum, _OVER)
for (int32_t i = 0; i < arbgroupIdNum; ++i) {
int32_t arbGroupId = 0;
SDB_GET_INT32(pRaw, dataPos, &arbGroupId, _OVER)
taosHashPut(pTrans->arbGroupIds, &arbGroupId, sizeof(int32_t), NULL, 0);
}
SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER)
terrno = 0;
@ -462,6 +484,9 @@ void mndTransDropData(STrans *pTrans) {
mndTransDropActions(pTrans->commitActions);
pTrans->commitActions = NULL;
}
if (pTrans->arbGroupIds != NULL) {
taosHashCleanup(pTrans->arbGroupIds);
}
if (pTrans->pRpcArray != NULL) {
taosArrayDestroy(pTrans->pRpcArray);
pTrans->pRpcArray = NULL;
@ -581,6 +606,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo));
pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64();
taosInitRWLatch(&pTrans->lockRpcArray);
@ -733,7 +759,9 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname)
}
}
void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId) { pTrans->arbGroupId = groupId; }
void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId) {
taosHashPut(pTrans->arbGroupIds, &groupId, sizeof(int32_t), NULL, 0);
}
void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }
@ -821,7 +849,16 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
if (pNew->conflict == TRN_CONFLICT_ARBGROUP) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) {
if (pNew->arbGroupId == pTrans->arbGroupId) conflict = true;
void *pIter = taosHashIterate(pNew->arbGroupIds, NULL);
while (pIter != NULL) {
int32_t groupId = *(int32_t *)pIter;
if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) {
taosHashCancelIterate(pNew->arbGroupIds, pIter);
conflict = true;
break;
}
pIter = taosHashIterate(pNew->arbGroupIds, pIter);
}
}
}
@ -1372,7 +1409,7 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr
mInfo("trans:%d, execute %d actions serial, current redoAction:%d", pTrans->id, numOfActions, pTrans->actionPos);
for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pActions, pTrans->actionPos);
STransAction *pAction = taosArrayGet(pActions, action);
code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf);
if (code == 0) {

10
source/dnode/mnode/impl/test/func/func.cpp
View File

@ -49,6 +49,7 @@ TEST_F(MndTestFunc, 01_Show_Func) {
}
TEST_F(MndTestFunc, 02_Create_Func) {
#ifndef WINDOWS
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "");
@ -159,9 +160,11 @@ TEST_F(MndTestFunc, 02_Create_Func) {
test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", "");
EXPECT_EQ(test.GetShowRows(), 1);
#endif
}
TEST_F(MndTestFunc, 03_Retrieve_Func) {
#ifndef WINDOWS
{
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
@ -376,9 +379,11 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST);
}
#endif
}
TEST_F(MndTestFunc, 04_Drop_Func) {
#ifndef WINDOWS
{
SDropFuncReq dropReq = {0};
strcpy(dropReq.name, "");
@ -441,9 +446,11 @@ TEST_F(MndTestFunc, 04_Drop_Func) {
test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", "");
EXPECT_EQ(test.GetShowRows(), 1);
#endif
}
TEST_F(MndTestFunc, 05_Actual_code) {
#ifndef WINDOWS
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "udf1");
@ -507,4 +514,5 @@ TEST_F(MndTestFunc, 05_Actual_code) {
}
tFreeSRetrieveFuncRsp(&retrieveRsp);
}
}
#endif
}

9
source/dnode/vnode/src/inc/tsdb.h
View File

@ -355,6 +355,8 @@ typedef struct {
int flush_count;
} SCacheFlushState;
typedef struct SCompMonitor SCompMonitor;
struct STsdb {
char * path;
SVnode * pVnode;
@ -375,8 +377,11 @@ struct STsdb {
TdThreadMutex pgMutex;
struct STFileSystem *pFS; // new
SRocksCache rCache;
// compact monitor
struct SCompMonitor *pCompMonitor;
SCompMonitor *pCompMonitor;
struct {
SVHashTable *ht;
SArray *arr;
} *commitInfo;
};
struct TSDBKEY {

28
source/dnode/vnode/src/inc/vnd.h
View File

@ -49,31 +49,21 @@ int32_t vnodeEncodeConfig(const void* pObj, SJson* pJson);
int32_t vnodeDecodeConfig(const SJson* pJson, void* pObj);
// vnodeAsync.c
typedef struct SVAsync SVAsync;
typedef enum {
EVA_PRIORITY_HIGH = 0,
EVA_PRIORITY_NORMAL,
EVA_PRIORITY_LOW,
} EVAPriority;
#define VNODE_ASYNC_VALID_CHANNEL_ID(channelId) ((channelId) > 0)
#define VNODE_ASYNC_VALID_TASK_ID(taskId) ((taskId) > 0)
int32_t vnodeAsyncInit(SVAsync** async, char* label);
int32_t vnodeAsyncDestroy(SVAsync** async);
int32_t vnodeAChannelInit(SVAsync* async, int64_t* channelId);
int32_t vnodeAChannelDestroy(SVAsync* async, int64_t channelId, bool waitRunning);
int32_t vnodeAsync(SVAsync* async, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), void* arg,
int64_t* taskId);
int32_t vnodeAsyncC(SVAsync* async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void*),
void (*complete)(void*), void* arg, int64_t* taskId);
int32_t vnodeAWait(SVAsync* async, int64_t taskId);
int32_t vnodeACancel(SVAsync* async, int64_t taskId);
int32_t vnodeAsyncSetWorkers(SVAsync* async, int32_t numWorkers);
// vnodeModule.c
extern SVAsync* vnodeAsyncHandle[2];
int32_t vnodeAsyncOpen(int32_t numOfThreads);
int32_t vnodeAsyncClose();
int32_t vnodeAChannelInit(int64_t async, SVAChannelID* channelID);
int32_t vnodeAChannelDestroy(SVAChannelID* channelID, bool waitRunning);
int32_t vnodeAsync(SVAChannelID* channelID, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*),
void* arg, SVATaskID* taskID);
int32_t vnodeAWait(SVATaskID* taskID);
int32_t vnodeACancel(SVATaskID* taskID);
int32_t vnodeAsyncSetWorkers(int64_t async, int32_t numWorkers);
// vnodeBufPool.c
typedef struct SVBufPoolNode SVBufPoolNode;

35
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -237,9 +237,6 @@ int32_t tsdbCacheNewSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, int8_t
int32_t tsdbCacheDropSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, bool hasPrimayKey);
int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type);
int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey);
int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync);
int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp);
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows);
@ -470,6 +467,16 @@ typedef struct SVMonitorObj {
taos_counter_t* insertCounter;
} SVMonitorObj;
typedef struct {
int64_t async;
int64_t id;
} SVAChannelID;
typedef struct {
int64_t async;
int64_t id;
} SVATaskID;
struct SVnode {
char* path;
SVnodeCfg config;
@ -491,8 +498,8 @@ struct SVnode {
SVBufPool* onRecycle;
// commit variables
int64_t commitChannel;
int64_t commitTask;
SVAChannelID commitChannel;
SVATaskID commitTask;
SMeta* pMeta;
SSma* pSma;
@ -598,6 +605,24 @@ struct SCompactInfo {
void initStorageAPI(SStorageAPI* pAPI);
// a simple hash table impl
typedef struct SVHashTable SVHashTable;
struct SVHashTable {
uint32_t (*hash)(const void*);
int32_t (*compare)(const void*, const void*);
int32_t numEntries;
uint32_t numBuckets;
struct SVHashEntry** buckets;
};
#define vHashNumEntries(ht) ((ht)->numEntries)
int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*));
int32_t vHashDestroy(SVHashTable** ht);
int32_t vHashPut(SVHashTable* ht, void* obj);
int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj);
int32_t vHashDrop(SVHashTable* ht, const void* obj);
#ifdef __cplusplus
}
#endif

4
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -622,8 +622,8 @@ int32_t smaRetention(SSma *pSma, int64_t now) {
for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
if (pSma->pRSmaTsdb[i]) {
code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1);
if (code) goto _end;
// code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1);
// if (code) goto _end;
}
}

24
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -588,6 +588,13 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud
tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud);
}
for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) {
SValue *pValue = &pLastCol->rowKey.pks[i];
if (IS_VAR_DATA_TYPE(pValue->type)) {
taosMemoryFree(pValue->pData);
}
}
if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type) /* && pLastCol->colVal.value.nData > 0*/) {
taosMemoryFree(pLastCol->colVal.value.pData);
}
@ -1072,6 +1079,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray
SLastCol *PToFree = pLastCol;
if (IS_LAST_KEY(idxKey->key) && !COL_VAL_IS_VALUE(pColVal)) {
taosMemoryFreeClear(PToFree);
rocksdb_free(values_list[i]);
continue;
}
@ -1238,6 +1247,18 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo
ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx));
// 1. prepare last
STsdbRowKey tsdbRowKey = {0};
tsdbRowGetKey(&lRow, &tsdbRowKey);
{
SLastUpdateCtx updateCtx = {
.lflag = LFLAG_LAST,
.tsdbRowKey = tsdbRowKey,
.colVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP,
.val = lRow.pBlockData->aTSKEY[lRow.iRow]}))};
taosArrayPush(ctxArray, &updateCtx);
}
TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0);
for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) {
@ -1263,9 +1284,6 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo
}
// 2. prepare last row
STsdbRowKey tsdbRowKey = {0};
tsdbRowGetKey(&lRow, &tsdbRowKey);
STSDBRowIter iter = {0};
tsdbRowIterOpen(&iter, &lRow, pTSchema);
for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) {

395
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -17,13 +17,12 @@
// extern dependencies
typedef struct {
STsdb *tsdb;
TFileSetArray *fsetArr;
TFileOpArray fopArray[1];
// SSkmInfo skmTb[1];
// SSkmInfo skmRow[1];
int32_t fid;
STFileSet *fset;
} SFileSetCommitInfo;
typedef struct {
STsdb *tsdb;
int32_t minutes;
int8_t precision;
int32_t minRow;
@ -32,34 +31,34 @@ typedef struct {
int32_t sttTrigger;
int32_t szPage;
int64_t compactVersion;
int64_t cid;
int64_t now;
struct {
int64_t cid;
int64_t now;
TSKEY nextKey;
int32_t fid;
int32_t expLevel;
SDiskID did;
TSKEY minKey;
TSKEY maxKey;
STFileSet *fset;
TABLEID tbid[1];
bool hasTSData;
bool skipTsRow;
SHashObj *pColCmprObj;
SFileSetCommitInfo *info;
int32_t expLevel;
SDiskID did;
TSKEY minKey;
TSKEY maxKey;
TABLEID tbid[1];
bool hasTSData;
bool skipTsRow;
SHashObj *pColCmprObj;
} ctx[1];
// reader
TSttFileReaderArray sttReaderArray[1];
// iter
TTsdbIterArray dataIterArray[1];
SIterMerger *dataIterMerger;
TTsdbIterArray tombIterArray[1];
SIterMerger *tombIterMerger;
// writer
SFSetWriter *writer;
TFileOpArray fopArray[1];
} SCommitter2;
static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
@ -74,8 +73,8 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
.maxRow = committer->maxRow,
.szPage = committer->szPage,
.cmprAlg = committer->cmprAlg,
.fid = committer->ctx->fid,
.cid = committer->ctx->cid,
.fid = committer->ctx->info->fid,
.cid = committer->cid,
.did = committer->ctx->did,
.level = 0,
};
@ -83,11 +82,11 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
if (committer->sttTrigger == 1) {
config.toSttOnly = false;
if (committer->ctx->fset) {
if (committer->ctx->info->fset) {
for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ftype++) {
if (committer->ctx->fset->farr[ftype] != NULL) {
if (committer->ctx->info->fset->farr[ftype] != NULL) {
config.files[ftype].exist = true;
config.files[ftype].file = committer->ctx->fset->farr[ftype]->f[0];
config.files[ftype].file = committer->ctx->info->fset->farr[ftype]->f[0];
}
}
}
@ -117,7 +116,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
committer->ctx->tbid->suid = 0;
committer->ctx->tbid->uid = 0;
for (SRowInfo *row; (row = tsdbIterMergerGetData(committer->dataIterMerger)) != NULL;) {
if (row->uid != committer->ctx->tbid->uid) {
committer->ctx->tbid->suid = row->suid;
@ -132,7 +130,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
int64_t ts = TSDBROW_TS(&row->row);
if (ts > committer->ctx->maxKey) {
committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts);
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
TSDB_CHECK_CODE(code, lino, _exit);
continue;
@ -152,7 +149,8 @@ _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
} else {
tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, numOfRow);
tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid,
numOfRow);
}
return code;
}
@ -168,7 +166,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
}
// do not need to write tomb data if there is no ts data
bool skip = (committer->ctx->fset == NULL && !committer->ctx->hasTSData);
bool skip = (committer->ctx->info->fset == NULL && !committer->ctx->hasTSData);
committer->ctx->tbid->suid = 0;
committer->ctx->tbid->uid = 0;
@ -187,12 +185,8 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
if (record->ekey < committer->ctx->minKey) {
// do nothing
} else if (record->skey > committer->ctx->maxKey) {
committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey);
// committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey);
} else {
if (record->ekey > committer->ctx->maxKey) {
committer->ctx->nextKey = TMIN(committer->ctx->nextKey, committer->ctx->maxKey + 1);
}
record->skey = TMAX(record->skey, committer->ctx->minKey);
record->ekey = TMIN(record->ekey, committer->ctx->maxKey);
@ -211,8 +205,8 @@ _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
} else {
tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), committer->ctx->fid,
numRecord);
tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode),
committer->ctx->info->fid, numRecord);
}
return code;
}
@ -223,15 +217,15 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
ASSERT(TARRAY2_SIZE(committer->sttReaderArray) == 0);
if (committer->ctx->fset == NULL //
|| committer->sttTrigger > 1 //
|| TARRAY2_SIZE(committer->ctx->fset->lvlArr) == 0 //
if (committer->ctx->info->fset == NULL //
|| committer->sttTrigger > 1 //
|| TARRAY2_SIZE(committer->ctx->info->fset->lvlArr) == 0 //
) {
return 0;
}
SSttLvl *lvl;
TARRAY2_FOREACH(committer->ctx->fset->lvlArr, lvl) {
TARRAY2_FOREACH(committer->ctx->info->fset->lvlArr, lvl) {
STFileObj *fobj = NULL;
TARRAY2_FOREACH(lvl->fobjArr, fobj) {
SSttFileReader *sttReader;
@ -289,7 +283,7 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
config.from->version = VERSION_MIN;
config.from->key = (SRowKey){
.ts = committer->ctx->minKey,
.numOfPKs = 0, // TODO: support multiple primary keys
.numOfPKs = 0,
};
code = tsdbIterOpen(&config, &iter);
@ -359,22 +353,15 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
int32_t lino = 0;
STsdb *tsdb = committer->tsdb;
int32_t fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision);
// check if can commit
tsdbFSCheckCommit(tsdb, fid);
tsdbFSCheckCommit(tsdb, committer->ctx->info->fid);
committer->ctx->fid = fid;
committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now);
tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now);
tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
&committer->ctx->maxKey);
code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did);
TSDB_CHECK_CODE(code, lino, _exit);
tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did);
STFileSet fset = {.fid = committer->ctx->fid};
committer->ctx->fset = &fset;
STFileSet **fsetPtr = TARRAY2_SEARCH(committer->fsetArr, &committer->ctx->fset, tsdbTFileSetCmprFn, TD_EQ);
committer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr;
committer->ctx->tbid->suid = 0;
committer->ctx->tbid->uid = 0;
@ -391,15 +378,13 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
code = tsdbCommitOpenWriter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
// reset nextKey
committer->ctx->nextKey = TSKEY_MAX;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
} else {
tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode),
__func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel);
__func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey,
committer->ctx->expLevel);
}
return code;
}
@ -421,7 +406,7 @@ _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
} else {
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid);
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
}
return code;
}
@ -449,7 +434,220 @@ _exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
} else {
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid);
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
}
return code;
}
static int32_t tFileSetCommitInfoCompare(const void *arg1, const void *arg2) {
SFileSetCommitInfo *info1 = (SFileSetCommitInfo *)arg1;
SFileSetCommitInfo *info2 = (SFileSetCommitInfo *)arg2;
if (info1->fid < info2->fid) {
return -1;
} else if (info1->fid > info2->fid) {
return 1;
} else {
return 0;
}
}
static int32_t tFileSetCommitInfoPCompare(const void *arg1, const void *arg2) {
return tFileSetCommitInfoCompare(*(SFileSetCommitInfo **)arg1, *(SFileSetCommitInfo **)arg2);
}
static uint32_t tFileSetCommitInfoHash(const void *arg) {
SFileSetCommitInfo *info = (SFileSetCommitInfo *)arg;
return MurmurHash3_32((const char *)&info->fid, sizeof(info->fid));
}
static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) {
int32_t code = 0;
int32_t lino = 0;
if (pTsdb->commitInfo) {
for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
vHashDrop(pTsdb->commitInfo->ht, info);
tsdbTFileSetClear(&info->fset);
taosMemoryFree(info);
}
vHashDestroy(&pTsdb->commitInfo->ht);
taosArrayDestroy(pTsdb->commitInfo->arr);
pTsdb->commitInfo->arr = NULL;
taosMemoryFreeClear(pTsdb->commitInfo);
}
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
}
static int32_t tsdbCommitInfoInit(STsdb *pTsdb) {
int32_t code = 0;
int32_t lino = 0;
pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo));
if (pTsdb->commitInfo == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
code = vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare);
TSDB_CHECK_CODE(code, lino, _exit);
pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *));
if (pTsdb->commitInfo->arr == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
_exit:
if (code) {
tsdbCommitInfoDestroy(pTsdb);
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
}
static int32_t tsdbCommitInfoAdd(STsdb *tsdb, int32_t fid) {
int32_t code = 0;
int32_t lino = 0;
SFileSetCommitInfo *tinfo;
if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
tinfo->fid = fid;
tinfo->fset = NULL;
code = vHashPut(tsdb->commitInfo->ht, tinfo);
TSDB_CHECK_CODE(code, lino, _exit);
if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
}
static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
int32_t code = 0;
int32_t lino = 0;
STFileSet *fset = NULL;
SRBTreeIter iter;
code = tsdbCommitInfoInit(tsdb);
TSDB_CHECK_CODE(code, lino, _exit);
// scan time-series data
iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) {
STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn);
// scan time-series data
STsdbRowKey from = {
.key.ts = INT64_MIN,
.key.numOfPKs = 0,
.version = INT64_MIN,
};
for (;;) {
int64_t minKey, maxKey;
STbDataIter tbDataIter = {0};
TSDBROW *row;
int32_t fid;
tsdbTbDataIterOpen(pTbData, &from, 0, &tbDataIter);
if ((row = tsdbTbDataIterGet(&tbDataIter)) == NULL) {
break;
}
fid = tsdbKeyFid(TSDBROW_TS(row), tsdb->keepCfg.days, tsdb->keepCfg.precision);
tsdbFidKeyRange(fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
SFileSetCommitInfo *info;
SFileSetCommitInfo tinfo = {
.fid = fid,
};
vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info);
if (info == NULL) {
code = tsdbCommitInfoAdd(tsdb, fid);
TSDB_CHECK_CODE(code, lino, _exit);
}
from.key.ts = maxKey + 1;
}
}
taosThreadMutexLock(&tsdb->mutex);
// scan tomb data
if (tsdb->imem->nDel > 0) {
TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
if (tsdbTFileSetIsEmpty(fset)) {
continue;
}
SFileSetCommitInfo *info;
SFileSetCommitInfo tinfo = {
.fid = fset->fid,
};
// check if the file set already on the commit list
vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info);
if (info != NULL) {
continue;
}
int64_t minKey, maxKey;
bool hasDataToCommit = false;
tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) {
STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn);
for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) {
if (pDelData->sKey > maxKey || pDelData->eKey < minKey) {
continue;
} else {
hasDataToCommit = true;
if ((code = tsdbCommitInfoAdd(tsdb, fset->fid))) {
taosThreadMutexUnlock(&tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
break;
}
}
if (hasDataToCommit) {
break;
}
}
}
}
// begin tasks on file set
for (int i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset);
if (fset) {
tsdbTFileSetInitCopy(tsdb, fset, &info->fset);
}
}
taosThreadMutexUnlock(&tsdb->mutex);
_exit:
if (code) {
tsdbCommitInfoDestroy(tsdb);
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
}
@ -458,11 +656,7 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
int32_t code = 0;
int32_t lino = 0;
memset(committer, 0, sizeof(committer[0]));
committer->tsdb = tsdb;
code = tsdbFSCreateCopySnapshot(tsdb->pFS, &committer->fsetArr);
TSDB_CHECK_CODE(code, lino, _exit);
committer->minutes = tsdb->keepCfg.days;
committer->precision = tsdb->keepCfg.precision;
committer->minRow = info->info.config.tsdbCfg.minRows;
@ -471,21 +665,11 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
committer->sttTrigger = info->info.config.sttTrigger;
committer->szPage = info->info.config.tsdbPageSize;
committer->compactVersion = INT64_MAX;
committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS);
committer->ctx->now = taosGetTimestampSec();
committer->cid = tsdbFSAllocEid(tsdb->pFS);
committer->now = taosGetTimestampSec();
committer->ctx->nextKey = tsdb->imem->minKey;
if (tsdb->imem->nDel > 0) {
SRBTreeIter iter[1] = {tRBTreeIterCreate(tsdb->imem->tbDataTree, 1)};
for (SRBTreeNode *node = tRBTreeIterNext(iter); node; node = tRBTreeIterNext(iter)) {
STbData *tbData = TCONTAINER_OF(node, STbData, rbtn);
for (SDelData *delData = tbData->pHead; delData; delData = delData->pNext) {
committer->ctx->nextKey = TMIN(committer->ctx->nextKey, delData->sKey);
}
}
}
code = tsdbCommitInfoBuild(tsdb);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
@ -516,14 +700,13 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
TARRAY2_DESTROY(committer->sttReaderArray, NULL);
TARRAY2_DESTROY(committer->fopArray, NULL);
TARRAY2_DESTROY(committer->sttReaderArray, NULL);
tsdbFSDestroyCopySnapshot(&committer->fsetArr);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino,
tstrerror(code), committer->ctx->cid);
tstrerror(code), committer->cid);
} else {
tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->cid);
tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid);
}
return code;
}
@ -553,17 +736,18 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
taosThreadMutexUnlock(&tsdb->mutex);
tsdbUnrefMemTable(imem, NULL, true);
} else {
SCommitter2 committer[1];
SCommitter2 committer = {0};
code = tsdbOpenCommitter(tsdb, info, committer);
code = tsdbOpenCommitter(tsdb, info, &committer);
TSDB_CHECK_CODE(code, lino, _exit);
while (committer->ctx->nextKey != TSKEY_MAX) {
code = tsdbCommitFileSet(committer);
for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
code = tsdbCommitFileSet(&committer);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbCloseCommitter(committer, code);
code = tsdbCloseCommitter(&committer, code);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -580,22 +764,33 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
int32_t code = 0;
int32_t lino = 0;
if (tsdb->imem == NULL) goto _exit;
if (tsdb->imem) {
SMemTable *pMemTable = tsdb->imem;
taosThreadMutexLock(&tsdb->mutex);
if ((code = tsdbFSEditCommit(tsdb->pFS))) {
taosThreadMutexUnlock(&tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
tsdb->imem = NULL;
for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
if (info->fset) {
tsdbFinishTaskOnFileSet(tsdb, info->fid);
}
}
SMemTable *pMemTable = tsdb->imem;
taosThreadMutexLock(&tsdb->mutex);
code = tsdbFSEditCommit(tsdb->pFS);
if (code) {
taosThreadMutexUnlock(&tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCommitInfoDestroy(tsdb);
tsdbUnrefMemTable(pMemTable, NULL, true);
}
tsdb->imem = NULL;
taosThreadMutexUnlock(&tsdb->mutex);
tsdbUnrefMemTable(pMemTable, NULL, true);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code));
} else {
tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
}
@ -611,6 +806,16 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) {
code = tsdbFSEditAbort(pTsdb->pFS);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&pTsdb->mutex);
for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
if (info->fset) {
tsdbFinishTaskOnFileSet(pTsdb, info->fid);
}
}
taosThreadMutexUnlock(&pTsdb->mutex);
tsdbCommitInfoDestroy(pTsdb);
_exit:
if (code) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
@ -618,4 +823,4 @@ _exit:
tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__);
}
return code;
}
}

127
source/dnode/vnode/src/tsdb/tsdbFS2.c
View File

@ -22,9 +22,6 @@
extern void remove_file(const char *fname);
#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT
#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1)
typedef struct STFileHashEntry {
struct STFileHashEntry *next;
char fname[TSDB_FILENAME_LEN];
@ -290,10 +287,8 @@ static int32_t commit_edit(STFileSystem *fs) {
current_fname(fs->tsdb, current, TSDB_FCURRENT);
if (fs->etype == TSDB_FEDIT_COMMIT) {
current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C);
} else if (fs->etype == TSDB_FEDIT_MERGE) {
current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
} else {
ASSERT(0);
current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
}
int32_t code;
@ -324,11 +319,8 @@ static int32_t abort_edit(STFileSystem *fs) {
if (fs->etype == TSDB_FEDIT_COMMIT) {
current_fname(fs->tsdb, fname, TSDB_FCURRENT_C);
} else if (fs->etype == TSDB_FEDIT_MERGE) {
current_fname(fs->tsdb, fname, TSDB_FCURRENT_M);
} else {
tsdbError("vgId:%d %s failed since invalid etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype);
ASSERT(0);
current_fname(fs->tsdb, fname, TSDB_FCURRENT_M);
}
int32_t code;
@ -767,9 +759,12 @@ extern int32_t tsdbStopAllCompTask(STsdb *tsdb);
int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
STFileSystem *fs = pTsdb->pFS;
TARRAY2(int64_t) channelArr = {0};
SArray *channelArray = taosArrayInit(0, sizeof(SVAChannelID));
if (channelArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosThreadMutexLock(&fs->tsdb->mutex);
taosThreadMutexLock(&pTsdb->mutex);
// disable
pTsdb->bgTaskDisabled = true;
@ -777,20 +772,23 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
// collect channel
STFileSet *fset;
TARRAY2_FOREACH(fs->fSetArr, fset) {
if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) {
TARRAY2_APPEND(&channelArr, fset->bgTaskChannel);
fset->bgTaskChannel = 0;
if (fset->channelOpened) {
taosArrayPush(channelArray, &fset->channel);
fset->channel = (SVAChannelID){0};
fset->mergeScheduled = false;
tsdbFSSetBlockCommit(fset, false);
fset->channelOpened = false;
}
fset->mergeScheduled = false;
tsdbFSSetBlockCommit(fset, false);
}
taosThreadMutexUnlock(&fs->tsdb->mutex);
taosThreadMutexUnlock(&pTsdb->mutex);
// destroy all channels
int64_t channel;
TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); }
TARRAY2_DESTROY(&channelArr, NULL);
for (int32_t i = 0; i < taosArrayGetSize(channelArray); i++) {
SVAChannelID *channel = taosArrayGet(channelArray, i);
vnodeAChannelDestroy(channel, true);
}
taosArrayDestroy(channelArray);
#ifdef TD_ENTERPRISE
tsdbStopAllCompTask(pTsdb);
@ -832,15 +830,10 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e
int32_t lino;
char current_t[TSDB_FILENAME_LEN];
switch (etype) {
case TSDB_FEDIT_COMMIT:
current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C);
break;
case TSDB_FEDIT_MERGE:
current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
break;
default:
ASSERT(0);
if (etype == TSDB_FEDIT_COMMIT) {
current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C);
} else {
current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
}
tsem_wait(&fs->canEdit);
@ -932,8 +925,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
arg->tsdb = fs->tsdb;
arg->fid = fset->fid;
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg,
NULL);
code = vnodeAsync(&fset->channel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, NULL);
TSDB_CHECK_CODE(code, lino, _exit);
fset->mergeScheduled = true;
}
@ -946,33 +938,6 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
}
}
// clear empty level and fset
int32_t i = 0;
while (i < TARRAY2_SIZE(fs->fSetArr)) {
STFileSet *fset = TARRAY2_GET(fs->fSetArr, i);
int32_t j = 0;
while (j < TARRAY2_SIZE(fset->lvlArr)) {
SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, j);
if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear);
} else {
j++;
}
}
if (tsdbTFileSetIsEmpty(fset)) {
if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) {
vnodeAChannelDestroy(vnodeAsyncHandle[1], fset->bgTaskChannel, false);
fset->bgTaskChannel = 0;
}
TARRAY2_REMOVE(fs->fSetArr, i, tsdbTFileSetClear);
} else {
i++;
}
}
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code));
@ -1211,3 +1176,47 @@ _out:
}
int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); }
int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) {
int16_t sttTrigger = tsdb->pVnode->config.sttTrigger;
tsdbFSGetFSet(tsdb->pFS, fid, fset);
if (sttTrigger == 1 && (*fset)) {
for (;;) {
if ((*fset)->taskRunning) {
(*fset)->numWaitTask++;
taosThreadCondWait(&(*fset)->beginTask, &tsdb->mutex);
tsdbFSGetFSet(tsdb->pFS, fid, fset);
ASSERT(fset != NULL);
(*fset)->numWaitTask--;
ASSERT((*fset)->numWaitTask >= 0);
} else {
(*fset)->taskRunning = true;
break;
}
}
tsdbInfo("vgId:%d begin task on file set:%d", TD_VID(tsdb->pVnode), fid);
}
return 0;
}
int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) {
int16_t sttTrigger = tsdb->pVnode->config.sttTrigger;
if (sttTrigger == 1) {
STFileSet *fset = NULL;
tsdbFSGetFSet(tsdb->pFS, fid, &fset);
if (fset != NULL && fset->taskRunning) {
fset->taskRunning = false;
if (fset->numWaitTask > 0) {
taosThreadCondSignal(&fset->beginTask);
}
tsdbInfo("vgId:%d finish task on file set:%d", TD_VID(tsdb->pVnode), fid);
}
}
return 0;
}

10
source/dnode/vnode/src/tsdb/tsdbFS2.h
View File

@ -24,7 +24,9 @@ extern "C" {
typedef enum {
TSDB_FEDIT_COMMIT = 1, //
TSDB_FEDIT_MERGE
TSDB_FEDIT_MERGE,
TSDB_FEDIT_COMPACT,
TSDB_FEDIT_RETENTION,
} EFEditT;
typedef enum {
@ -59,6 +61,8 @@ int32_t tsdbFSEditAbort(STFileSystem *fs);
// other
int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset);
int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid);
int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset);
int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid);
// utils
int32_t save_fs(const TFileSetArray *arr, const char *fname);
int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype);
@ -72,10 +76,6 @@ struct STFileSystem {
EFEditT etype;
TFileSetArray fSetArr[1];
TFileSetArray fSetArrTmp[1];
// background task queue
bool stop;
int64_t taskid;
};
#ifdef __cplusplus

43
source/dnode/vnode/src/tsdb/tsdbFSet2.c
View File

@ -456,13 +456,14 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) {
TARRAY2_INIT(fset[0]->lvlArr);
// background task queue
fset[0]->bgTaskChannel = 0;
fset[0]->mergeScheduled = false;
taosThreadCondInit(&(*fset)->beginTask, NULL);
(*fset)->taskRunning = false;
(*fset)->numWaitTask = 0;
// block commit variables
taosThreadCondInit(&fset[0]->canCommit, NULL);
fset[0]->numWaitCommit = 0;
fset[0]->blockCommit = false;
(*fset)->numWaitCommit = 0;
(*fset)->blockCommit = false;
return 0;
}
@ -598,21 +599,19 @@ int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr) {
return 0;
}
int32_t tsdbTFileSetClear(STFileSet **fset) {
if (!fset[0]) return 0;
void tsdbTFileSetClear(STFileSet **fset) {
if (fset && *fset) {
for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
if ((*fset)->farr[ftype] == NULL) continue;
tsdbTFileObjUnref((*fset)->farr[ftype]);
}
for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
if (fset[0]->farr[ftype] == NULL) continue;
tsdbTFileObjUnref(fset[0]->farr[ftype]);
TARRAY2_DESTROY((*fset)->lvlArr, tsdbSttLvlClear);
taosThreadCondDestroy(&(*fset)->beginTask);
taosThreadCondDestroy(&(*fset)->canCommit);
taosMemoryFreeClear(*fset);
}
TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear);
taosThreadCondDestroy(&fset[0]->canCommit);
taosMemoryFree(fset[0]);
fset[0] = NULL;
return 0;
}
int32_t tsdbTFileSetRemove(STFileSet *fset) {
@ -665,6 +664,12 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) {
}
int32_t tsdbTFileSetOpenChannel(STFileSet *fset) {
if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0;
return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel);
int32_t code;
if (!fset->channelOpened) {
if ((code = vnodeAChannelInit(2, &fset->channel))) {
return code;
}
fset->channelOpened = true;
}
return 0;
}

14
source/dnode/vnode/src/tsdb/tsdbFSet2.h
View File

@ -42,7 +42,7 @@ typedef enum {
int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset);
int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
int32_t tsdbTFileSetClear(STFileSet **fset);
void tsdbTFileSetClear(STFileSet **fset);
int32_t tsdbTFileSetRemove(STFileSet *fset);
int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset,
@ -90,9 +90,15 @@ struct STFileSet {
STFileObj *farr[TSDB_FTYPE_MAX]; // file array
TSttLvlArray lvlArr[1]; // level array
// background task channel
int64_t bgTaskChannel;
bool mergeScheduled;
// background task
bool channelOpened;
SVAChannelID channel;
bool mergeScheduled;
// sttTrigger = 1
TdThreadCond beginTask;
bool taskRunning;
int32_t numWaitTask;
// block commit variables
TdThreadCond canCommit;

1
source/dnode/vnode/src/tsdb/tsdbMerge.c
View File

@ -593,5 +593,6 @@ _exit:
exit(EXIT_FAILURE);
}
tsdbTFileSetClear(&merger->fset);
taosMemoryFree(arg);
return code;
}

303
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -24,8 +24,8 @@ typedef struct {
int64_t now;
int64_t cid;
TFileSetArray *fsetArr;
TFileOpArray fopArr[1];
STFileSet *fset;
TFileOpArray fopArr;
} SRTNer;
static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
@ -35,7 +35,7 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
.of = fobj->f[0],
};
return TARRAY2_APPEND(rtner->fopArr, op);
return TARRAY2_APPEND(&rtner->fopArr, op);
}
static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
@ -71,7 +71,7 @@ static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFi
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
if (fdFrom) taosCloseFile(&fdFrom);
if (fdTo) taosCloseFile(&fdTo);
}
@ -108,7 +108,7 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
}
@ -128,7 +128,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
code = TARRAY2_APPEND(&rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
@ -152,7 +152,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
},
};
code = TARRAY2_APPEND(rtner->fopArr, op);
code = TARRAY2_APPEND(&rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// do copy the file
@ -167,7 +167,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
}
return code;
}
@ -176,80 +176,51 @@ typedef struct {
STsdb *tsdb;
int64_t now;
int32_t fid;
bool s3Migrate;
} SRtnArg;
static int32_t tsdbDoRetentionBegin(SRtnArg *arg, SRTNer *rtner) {
int32_t code = 0;
int32_t lino = 0;
STsdb *tsdb = arg->tsdb;
rtner->tsdb = tsdb;
rtner->szPage = tsdb->pVnode->config.tsdbPageSize;
rtner->now = arg->now;
rtner->cid = tsdbFSAllocEid(tsdb->pFS);
code = tsdbFSCreateCopySnapshot(tsdb->pFS, &rtner->fsetArr);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
} else {
tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__);
}
return code;
}
static int32_t tsdbDoRetentionEnd(SRTNer *rtner) {
int32_t code = 0;
int32_t lino = 0;
if (TARRAY2_SIZE(rtner->fopArr) == 0) goto _exit;
code = tsdbFSEditBegin(rtner->tsdb->pFS, rtner->fopArr, TSDB_FEDIT_MERGE);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&rtner->tsdb->mutex);
code = tsdbFSEditCommit(rtner->tsdb->pFS);
if (code) {
taosThreadMutexUnlock(&rtner->tsdb->mutex);
if (TARRAY2_SIZE(&rtner->fopArr) > 0) {
code = tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&rtner->tsdb->mutex);
code = tsdbFSEditCommit(rtner->tsdb->pFS);
if (code) {
taosThreadMutexUnlock(&rtner->tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosThreadMutexUnlock(&rtner->tsdb->mutex);
TARRAY2_DESTROY(&rtner->fopArr, NULL);
}
taosThreadMutexUnlock(&rtner->tsdb->mutex);
TARRAY2_DESTROY(rtner->fopArr, NULL);
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
} else {
tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__);
}
tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
return code;
}
static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
static int32_t tsdbDoRetention(SRTNer *rtner) {
int32_t code = 0;
int32_t lino = 0;
STFileObj *fobj = NULL;
STFileSet *fset = rtner->fset;
int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now);
if (expLevel < 0) { // remove the fileset
for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) {
if (fobj == NULL) continue;
/*
int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) {
code = tsdbRemoveFileObjectS3(rtner, fobj);
TSDB_CHECK_CODE(code, lino, _exit);
} else {*/
code = tsdbDoRemoveFileObject(rtner, fobj);
TSDB_CHECK_CODE(code, lino, _exit);
//}
}
SSttLvl *lvl;
@ -275,10 +246,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
if (fobj == NULL) continue;
if (fobj->f->did.level == did.level) {
/*
code = tsdbCheckMigrateS3(rtner, fobj, ftype, &did);
TSDB_CHECK_CODE(code, lino, _exit);
*/
continue;
}
@ -306,91 +273,110 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
}
return code;
}
static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); }
static void tsdbRetentionCancel(void *arg) { taosMemoryFree(arg); }
static int32_t tsdbDoRetentionAsync(void *arg) {
static int32_t tsdbDoS3Migrate(SRTNer *rtner);
static int32_t tsdbRetention(void *arg) {
int32_t code = 0;
int32_t lino = 0;
SRTNer rtner[1] = {0};
code = tsdbDoRetentionBegin(arg, rtner);
TSDB_CHECK_CODE(code, lino, _exit);
SRtnArg *rtnArg = (SRtnArg *)arg;
STsdb *pTsdb = rtnArg->tsdb;
SVnode *pVnode = pTsdb->pVnode;
STFileSet *fset = NULL;
SRTNer rtner = {
.tsdb = pTsdb,
.szPage = pVnode->config.tsdbPageSize,
.now = rtnArg->now,
.cid = tsdbFSAllocEid(pTsdb->pFS),
};
STFileSet *fset;
TARRAY2_FOREACH(rtner->fsetArr, fset) {
if (fset->fid != ((SRtnArg *)arg)->fid) continue;
// begin task
taosThreadMutexLock(&pTsdb->mutex);
tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, &fset);
if (fset && (code = tsdbTFileSetInitCopy(pTsdb, fset, &rtner.fset))) {
taosThreadMutexUnlock(&pTsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosThreadMutexUnlock(&pTsdb->mutex);
code = tsdbDoRetentionOnFileSet(rtner, fset);
// do retention
if (rtner.fset) {
if (rtnArg->s3Migrate) {
code = tsdbDoS3Migrate(&rtner);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
code = tsdbDoRetention(&rtner);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbDoRetentionEnd(&rtner);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbDoRetentionEnd(rtner);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
if (TARRAY2_DATA(rtner->fopArr)) {
TARRAY2_DESTROY(rtner->fopArr, NULL);
}
TFileSetArray **fsetArr = &rtner->fsetArr;
if (fsetArr[0]) {
tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
}
if (rtner.fset) {
taosThreadMutexLock(&pTsdb->mutex);
tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid);
taosThreadMutexUnlock(&pTsdb->mutex);
}
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
// clear resources
tsdbTFileSetClear(&rtner.fset);
TARRAY2_DESTROY(&rtner.fopArr, NULL);
taosMemoryFree(arg);
if (code) {
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(((SRtnArg *)arg)->tsdb->pVnode), __func__, code, lino);
}
return code;
}
int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate) {
int32_t code = 0;
taosThreadMutexLock(&tsdb->mutex);
if (tsdb->bgTaskDisabled) {
taosThreadMutexUnlock(&tsdb->mutex);
return 0;
}
int32_t lino = 0;
STFileSet *fset;
TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
code = tsdbTFileSetOpenChannel(fset);
if (code) {
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}
SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
if (arg == NULL) {
taosThreadMutexUnlock(&tsdb->mutex);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (!tsdb->bgTaskDisabled) {
TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
code = tsdbTFileSetOpenChannel(fset);
TSDB_CHECK_CODE(code, lino, _exit);
arg->tsdb = tsdb;
arg->now = now;
arg->fid = fset->fid;
SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
if (arg == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
if (sync) {
code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync,
tsdbFreeRtnArg, arg, NULL);
} else {
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync,
tsdbFreeRtnArg, arg, NULL);
}
if (code) {
tsdbFreeRtnArg(arg);
taosThreadMutexUnlock(&tsdb->mutex);
return code;
arg->tsdb = tsdb;
arg->now = now;
arg->fid = fset->fid;
arg->s3Migrate = s3Migrate;
if ((code = vnodeAsync(&fset->channel, EVA_PRIORITY_LOW, tsdbRetention, tsdbRetentionCancel, arg, NULL))) {
taosMemoryFree(arg);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
}
taosThreadMutexUnlock(&tsdb->mutex);
_exit:
if (code) {
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(tsdb->pVnode), __func__, code, lino);
}
return code;
}
int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now) {
int32_t code = 0;
taosThreadMutexLock(&tsdb->mutex);
code = tsdbAsyncRetentionImpl(tsdb, now, false);
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}
@ -462,7 +448,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
code = TARRAY2_APPEND(&rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
@ -486,7 +472,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
},
};
code = TARRAY2_APPEND(rtner->fopArr, op);
code = TARRAY2_APPEND(&rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
char fname[TSDB_FILENAME_LEN];
@ -566,7 +552,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
.of = fobj->f[0],
};
code = TARRAY2_APPEND(rtner->fopArr, op);
code = TARRAY2_APPEND(&rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
// create new
@ -590,7 +576,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
},
};
code = TARRAY2_APPEND(rtner->fopArr, op);
code = TARRAY2_APPEND(&rtner->fopArr, op);
TSDB_CHECK_CODE(code, lino, _exit);
char fname[TSDB_FILENAME_LEN];
@ -653,10 +639,11 @@ _exit:
return code;
}
static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) {
static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
int32_t code = 0;
int32_t lino = 0;
STFileSet *fset = rtner->fset;
STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
if (!fobj) return code;
@ -720,41 +707,7 @@ _exit:
return code;
}
static int32_t tsdbDoS3MigrateAsync(void *arg) {
int32_t code = 0;
int32_t lino = 0;
SRTNer rtner[1] = {0};
code = tsdbDoRetentionBegin(arg, rtner);
TSDB_CHECK_CODE(code, lino, _exit);
STFileSet *fset;
TARRAY2_FOREACH(rtner->fsetArr, fset) {
if (fset->fid != ((SRtnArg *)arg)->fid) continue;
code = tsdbDoS3MigrateOnFileSet(rtner, fset);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbDoRetentionEnd(rtner);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
if (TARRAY2_DATA(rtner->fopArr)) {
TARRAY2_DESTROY(rtner->fopArr, NULL);
}
TFileSetArray **fsetArr = &rtner->fsetArr;
if (fsetArr[0]) {
tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
}
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
}
return code;
}
int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) {
int32_t code = 0;
extern int8_t tsS3EnabledCfg;
@ -772,45 +725,7 @@ int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
}
taosThreadMutexLock(&tsdb->mutex);
if (tsdb->bgTaskDisabled) {
taosThreadMutexUnlock(&tsdb->mutex);
return 0;
}
STFileSet *fset;
TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
code = tsdbTFileSetOpenChannel(fset);
if (code) {
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}
SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
if (arg == NULL) {
taosThreadMutexUnlock(&tsdb->mutex);
return TSDB_CODE_OUT_OF_MEMORY;
}
arg->tsdb = tsdb;
arg->now = now;
arg->fid = fset->fid;
if (sync) {
code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
tsdbFreeRtnArg, arg, NULL);
} else {
code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
tsdbFreeRtnArg, arg, NULL);
}
if (code) {
tsdbFreeRtnArg(arg);
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}
}
code = tsdbAsyncRetentionImpl(tsdb, now, true);
taosThreadMutexUnlock(&tsdb->mutex);
return code;
}

169
source/dnode/vnode/src/vnd/vnodeAsync.c
View File

@ -16,6 +16,7 @@
#include "vnd.h"
#include "vnodeHash.h"
typedef struct SVAsync SVAsync;
typedef struct SVATask SVATask;
typedef struct SVAChannel SVAChannel;
@ -54,7 +55,7 @@ struct SVATask {
int32_t priorScore;
SVAChannel *channel;
int32_t (*execute)(void *);
void (*complete)(void *);
void (*cancel)(void *);
void *arg;
EVATaskState state;
@ -67,6 +68,11 @@ struct SVATask {
struct SVATask *next;
};
typedef struct {
void (*cancel)(void *);
void *arg;
} SVATaskCancelInfo;
#define VATASK_PIORITY(task_) ((task_)->priority - ((task_)->priorScore / 4))
// async channel
@ -112,6 +118,10 @@ struct SVAsync {
SVHashTable *taskTable;
};
SVAsync *vnodeAsyncs[3];
#define MIN_ASYNC_ID 1
#define MAX_ASYNC_ID (sizeof(vnodeAsyncs) / sizeof(vnodeAsyncs[0]) - 1)
static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
int32_t ret;
@ -160,11 +170,6 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
}
async->numTasks--;
// call complete callback
if (task->complete) {
task->complete(task->arg);
}
if (task->numWait == 0) {
taosThreadCondDestroy(&task->waitCond);
taosMemoryFree(task);
@ -176,7 +181,7 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
return 0;
}
static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
static int32_t vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) {
while (async->queue[0].next != &async->queue[0] || async->queue[1].next != &async->queue[1] ||
async->queue[2].next != &async->queue[2]) {
for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) {
@ -184,6 +189,12 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
SVATask *task = async->queue[i].next;
task->prev->next = task->next;
task->next->prev = task->prev;
if (task->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel,
.arg = task->arg,
});
}
vnodeAsyncTaskDone(async, task);
}
}
@ -194,6 +205,7 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
static void *vnodeAsyncLoop(void *arg) {
SVWorker *worker = (SVWorker *)arg;
SVAsync *async = worker->async;
SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo));
setThreadName(async->label);
@ -209,12 +221,12 @@ static void *vnodeAsyncLoop(void *arg) {
for (;;) {
if (async->stop || worker->workerId >= async->numWorkers) {
if (async->stop) { // cancel all tasks
vnodeAsyncCancelAllTasks(async);
vnodeAsyncCancelAllTasks(async, cancelArray);
}
worker->state = EVA_WORKER_STATE_STOP;
async->numLaunchWorkers--;
taosThreadMutexUnlock(&async->mutex);
return NULL;
goto _exit;
}
for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) {
@ -259,6 +271,12 @@ static void *vnodeAsyncLoop(void *arg) {
worker->runningTask->execute(worker->runningTask->arg);
}
_exit:
for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) {
SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i);
cancel->cancel(cancel->arg);
}
taosArrayDestroy(cancelArray);
return NULL;
}
@ -294,7 +312,7 @@ static int32_t vnodeAsyncChannelCompare(const void *obj1, const void *obj2) {
return 0;
}
int32_t vnodeAsyncInit(SVAsync **async, char *label) {
static int32_t vnodeAsyncInit(SVAsync **async, const char *label) {
int32_t ret;
if (async == NULL) {
@ -360,7 +378,7 @@ int32_t vnodeAsyncInit(SVAsync **async, char *label) {
return 0;
}
int32_t vnodeAsyncDestroy(SVAsync **async) {
static int32_t vnodeAsyncDestroy(SVAsync **async) {
if ((*async) == NULL) {
return TSDB_CODE_INVALID_PARA;
}
@ -433,20 +451,39 @@ static int32_t vnodeAsyncLaunchWorker(SVAsync *async) {
return 0;
}
#ifdef BUILD_NO_CALL
int32_t vnodeAsync(SVAsync *async, EVAPriority priority, int32_t (*execute)(void *), void (*complete)(void *),
void *arg, int64_t *taskId) {
return vnodeAsyncC(async, 0, priority, execute, complete, arg, taskId);
}
#endif
int32_t vnodeAsyncOpen(int32_t numOfThreads) {
int32_t code = 0;
int32_t lino = 0;
int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void *),
void (*complete)(void *), void *arg, int64_t *taskId) {
if (async == NULL || execute == NULL || channelId < 0) {
// vnode-commit
code = vnodeAsyncInit(&vnodeAsyncs[1], "vnode-commit");
TSDB_CHECK_CODE(code, lino, _exit);
vnodeAsyncSetWorkers(1, numOfThreads);
// vnode-merge
code = vnodeAsyncInit(&vnodeAsyncs[2], "vnode-merge");
TSDB_CHECK_CODE(code, lino, _exit);
vnodeAsyncSetWorkers(2, numOfThreads);
_exit:
return 0;
}
int32_t vnodeAsyncClose() {
vnodeAsyncDestroy(&vnodeAsyncs[1]);
vnodeAsyncDestroy(&vnodeAsyncs[2]);
return 0;
}
int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*execute)(void *), void (*cancel)(void *),
void *arg, SVATaskID *taskID) {
if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || execute == NULL ||
channelID->id < 0) {
return TSDB_CODE_INVALID_PARA;
}
int64_t id;
int64_t id;
SVAsync *async = vnodeAsyncs[channelID->async];
// create task object
SVATask *task = (SVATask *)taosMemoryCalloc(1, sizeof(SVATask));
@ -457,7 +494,7 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int
task->priority = priority;
task->priorScore = 0;
task->execute = execute;
task->complete = complete;
task->cancel = cancel;
task->arg = arg;
task->state = EVA_TASK_STATE_WAITTING;
task->numWait = 0;
@ -466,10 +503,12 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int
// schedule task
taosThreadMutexLock(&async->mutex);
if (channelId == 0) {
if (channelID->id == 0) {
task->channel = NULL;
} else {
SVAChannel channel = {.channelId = channelId};
SVAChannel channel = {
.channelId = channelID->id,
};
vHashGet(async->channelTable, &channel, (void **)&task->channel);
if (task->channel == NULL) {
taosThreadMutexUnlock(&async->mutex);
@ -540,20 +579,24 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int
taosThreadMutexUnlock(&async->mutex);
if (taskId != NULL) {
*taskId = id;
if (taskID != NULL) {
taskID->async = channelID->async;
taskID->id = id;
}
return 0;
}
int32_t vnodeAWait(SVAsync *async, int64_t taskId) {
if (async == NULL || taskId <= 0) {
int32_t vnodeAWait(SVATaskID *taskID) {
if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) {
return TSDB_CODE_INVALID_PARA;
}
SVAsync *async = vnodeAsyncs[taskID->async];
SVATask *task = NULL;
SVATask task2 = {.taskId = taskId};
SVATask task2 = {
.taskId = taskID->id,
};
taosThreadMutexLock(&async->mutex);
@ -574,21 +617,27 @@ int32_t vnodeAWait(SVAsync *async, int64_t taskId) {
return 0;
}
int32_t vnodeACancel(SVAsync *async, int64_t taskId) {
if (async == NULL) {
int32_t vnodeACancel(SVATaskID *taskID) {
if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) {
return TSDB_CODE_INVALID_PARA;
}
int32_t ret = 0;
SVAsync *async = vnodeAsyncs[taskID->async];
SVATask *task = NULL;
SVATask task2 = {.taskId = taskId};
SVATask task2 = {
.taskId = taskID->id,
};
void (*cancel)(void *) = NULL;
void *arg = NULL;
taosThreadMutexLock(&async->mutex);
vHashGet(async->taskTable, &task2, (void **)&task);
if (task) {
if (task->state == EVA_TASK_STATE_WAITTING) {
// remove from queue
cancel = task->cancel;
arg = task->arg;
task->next->prev = task->prev;
task->prev->next = task->next;
vnodeAsyncTaskDone(async, task);
@ -599,14 +648,18 @@ int32_t vnodeACancel(SVAsync *async, int64_t taskId) {
taosThreadMutexUnlock(&async->mutex);
if (cancel) {
cancel(arg);
}
return ret;
}
int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) {
if (async == NULL || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) {
int32_t vnodeAsyncSetWorkers(int64_t asyncID, int32_t numWorkers) {
if (asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) {
return TSDB_CODE_INVALID_PARA;
}
SVAsync *async = vnodeAsyncs[asyncID];
taosThreadMutexLock(&async->mutex);
async->numWorkers = numWorkers;
if (async->numIdleWorkers > 0) {
@ -617,11 +670,13 @@ int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) {
return 0;
}
int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) {
if (async == NULL || channelId == NULL) {
int32_t vnodeAChannelInit(int64_t asyncID, SVAChannelID *channelID) {
if (channelID == NULL || asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID) {
return TSDB_CODE_INVALID_PARA;
}
SVAsync *async = vnodeAsyncs[asyncID];
// create channel object
SVAChannel *channel = (SVAChannel *)taosMemoryMalloc(sizeof(SVAChannel));
if (channel == NULL) {
@ -637,7 +692,7 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) {
// register channel
taosThreadMutexLock(&async->mutex);
channel->channelId = *channelId = ++async->nextChannelId;
channel->channelId = channelID->id = ++async->nextChannelId;
// add to hash table
int32_t ret = vHashPut(async->channelTable, channel);
@ -657,16 +712,24 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) {
taosThreadMutexUnlock(&async->mutex);
channelID->async = asyncID;
return 0;
}
int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning) {
if (async == NULL || channelId <= 0) {
int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || channelID->id <= 0) {
return TSDB_CODE_INVALID_PARA;
}
SVAsync *async = vnodeAsyncs[channelID->async];
SVAChannel *channel = NULL;
SVAChannel channel2 = {.channelId = channelId};
SVAChannel channel2 = {
.channelId = channelID->id,
};
SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo));
if (cancelArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosThreadMutexLock(&async->mutex);
@ -684,6 +747,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning
SVATask *task = channel->queue[i].next;
task->prev->next = task->next;
task->next->prev = task->prev;
if (task->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel,
.arg = task->arg,
});
}
vnodeAsyncTaskDone(async, task);
}
}
@ -693,6 +762,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning
if (channel->scheduled) {
channel->scheduled->prev->next = channel->scheduled->next;
channel->scheduled->next->prev = channel->scheduled->prev;
if (channel->scheduled->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = channel->scheduled->cancel,
.arg = channel->scheduled->arg,
});
}
vnodeAsyncTaskDone(async, channel->scheduled);
}
taosMemoryFree(channel);
@ -713,12 +788,16 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning
channel->state = EVA_CHANNEL_STATE_CLOSE;
}
}
} else {
taosThreadMutexUnlock(&async->mutex);
return TSDB_CODE_INVALID_PARA;
}
taosThreadMutexUnlock(&async->mutex);
for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) {
SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i);
cancel->cancel(cancel->arg);
}
taosArrayDestroy(cancelArray);
channelID->async = 0;
channelID->id = 0;
return 0;
}

32
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -289,7 +289,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
int64_t lastCommitted = pInfo->info.state.committed;
// wait last commit task
vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask);
vnodeAWait(&pVnode->commitTask);
if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit;
@ -361,15 +361,14 @@ static void vnodeReturnBufPool(SVnode *pVnode) {
taosThreadMutexUnlock(&pVnode->mutex);
}
static int32_t vnodeCommitTask(void *arg) {
static int32_t vnodeCommit(void *arg) {
int32_t code = 0;
SCommitInfo *pInfo = (SCommitInfo *)arg;
SVnode *pVnode = pInfo->pVnode;
// commit
code = vnodeCommitImpl(pInfo);
if (code) {
if ((code = vnodeCommitImpl(pInfo))) {
vFatal("vgId:%d, failed to commit vnode since %s", TD_VID(pVnode), terrstr());
taosMsleep(100);
exit(EXIT_FAILURE);
@ -379,37 +378,34 @@ static int32_t vnodeCommitTask(void *arg) {
vnodeReturnBufPool(pVnode);
_exit:
taosMemoryFree(arg);
return code;
}
static void vnodeCompleteCommit(void *arg) { taosMemoryFree(arg); }
static void vnodeCommitCancel(void *arg) { taosMemoryFree(arg); }
int vnodeAsyncCommit(SVnode *pVnode) {
int32_t code = 0;
int32_t lino = 0;
SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo));
if (NULL == pInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
// prepare to commit
code = vnodePrepareCommit(pVnode, pInfo);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
TSDB_CHECK_CODE(code, lino, _exit);
// schedule the task
code = vnodeAsyncC(vnodeAsyncHandle[0], pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommitTask,
vnodeCompleteCommit, pInfo, &pVnode->commitTask);
code =
vnodeAsync(&pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommit, vnodeCommitCancel, pInfo, &pVnode->commitTask);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
if (NULL != pInfo) {
taosMemoryFree(pInfo);
}
vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code),
pVnode->state.commitID);
taosMemoryFree(pInfo);
vError("vgId:%d %s failed at line %d since %s" PRId64, TD_VID(pVnode), __func__, lino, tstrerror(code));
} else {
vInfo("vgId:%d, vnode async commit done, commitId:%" PRId64 " term:%" PRId64 " applied:%" PRId64, TD_VID(pVnode),
pVnode->state.commitID, pVnode->state.applyTerm, pVnode->state.applied);
@ -419,7 +415,7 @@ _exit:
int vnodeSyncCommit(SVnode *pVnode) {
vnodeAsyncCommit(pVnode);
vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask);
vnodeAWait(&pVnode->commitTask);
return 0;
}

17
source/dnode/vnode/src/vnd/vnodeHash.h
View File

@ -22,23 +22,6 @@
extern "C" {
#endif
typedef struct SVHashTable SVHashTable;
struct SVHashTable {
uint32_t (*hash)(const void*);
int32_t (*compare)(const void*, const void*);
int32_t numEntries;
uint32_t numBuckets;
struct SVHashEntry** buckets;
};
#define vHashNumEntries(ht) ((ht)->numEntries)
int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*));
int32_t vHashDestroy(SVHashTable** ht);
int32_t vHashPut(SVHashTable* ht, void* obj);
int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj);
int32_t vHashDrop(SVHashTable* ht, const void* obj);
#ifdef __cplusplus
}
#endif

15
source/dnode/vnode/src/vnd/vnodeModule.c
View File

@ -18,8 +18,6 @@
static volatile int32_t VINIT = 0;
SVAsync* vnodeAsyncHandle[2];
int vnodeInit(int nthreads) {
int32_t init;
@ -28,13 +26,9 @@ int vnodeInit(int nthreads) {
return 0;
}
// vnode-commit
vnodeAsyncInit(&vnodeAsyncHandle[0], "vnode-commit");
vnodeAsyncSetWorkers(vnodeAsyncHandle[0], nthreads);
// vnode-merge
vnodeAsyncInit(&vnodeAsyncHandle[1], "vnode-merge");
vnodeAsyncSetWorkers(vnodeAsyncHandle[1], nthreads);
if (vnodeAsyncOpen(nthreads) != 0) {
return -1;
}
if (walInit() < 0) {
return -1;
@ -48,8 +42,7 @@ void vnodeCleanup() {
if (init == 0) return;
// set stop
vnodeAsyncDestroy(&vnodeAsyncHandle[0]);
vnodeAsyncDestroy(&vnodeAsyncHandle[1]);
vnodeAsyncClose();
walCleanUp();
smaCleanUp();

6
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -399,7 +399,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
taosThreadMutexInit(&pVnode->mutex, NULL);
taosThreadCondInit(&pVnode->poolNotEmpty, NULL);
if (vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel) != 0) {
if (vnodeAChannelInit(1, &pVnode->commitChannel) != 0) {
vError("vgId:%d, failed to init commit channel", TD_VID(pVnode));
goto _err;
}
@ -527,8 +527,8 @@ void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); }
void vnodeClose(SVnode *pVnode) {
if (pVnode) {
vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask);
vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true);
vnodeAWait(&pVnode->commitTask);
vnodeAChannelDestroy(&pVnode->commitChannel, true);
vnodeSyncClose(pVnode);
vnodeQueryClose(pVnode);
tqClose(pVnode->pTq);

21
source/dnode/vnode/src/vnd/vnodeRetention.c
View File

@ -15,20 +15,15 @@
#include "vnd.h"
int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) {
int32_t code = TSDB_CODE_SUCCESS;
extern int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now);
extern int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now);
code = tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now);
return code;
int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now) {
// async retention
return tsdbAsyncRetention(pVnode->pTsdb, now);
}
int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now) {
int32_t code = TSDB_CODE_SUCCESS;
code = tsdbS3Migrate(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
return code;
int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now) {
// async migration
return tsdbAsyncS3Migrate(pVnode->pTsdb, now);
}

4
source/dnode/vnode/src/vnd/vnodeSnapshot.c
View File

@ -622,13 +622,13 @@ extern int32_t tsdbEnableBgTask(STsdb *pTsdb);
static int32_t vnodeCancelAndDisableAllBgTask(SVnode *pVnode) {
tsdbDisableAndCancelAllBgTask(pVnode->pTsdb);
vnodeSyncCommit(pVnode);
vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true);
vnodeAChannelDestroy(&pVnode->commitChannel, true);
return 0;
}
static int32_t vnodeEnableBgTask(SVnode *pVnode) {
tsdbEnableBgTask(pVnode->pTsdb);
vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel);
vnodeAChannelInit(1, &pVnode->commitChannel);
return 0;
}

32
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -50,7 +50,7 @@ static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t l
static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp);
static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
extern int32_t vnodeProcessKillCompactReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
extern int32_t vnodeQueryCompactProgress(SVnode *pVnode, SRpcMsg *pMsg);
@ -866,7 +866,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) {
pMetaRsp->precision = pVnode->config.tsdbCfg.precision;
}
extern int32_t vnodeDoRetention(SVnode *pVnode, int64_t now);
extern int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now);
static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0;
@ -880,13 +880,13 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int3
vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp);
code = vnodeDoRetention(pVnode, trimReq.timestamp);
code = vnodeAsyncRetention(pVnode, trimReq.timestamp);
_exit:
return code;
}
extern int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now);
extern int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now);
static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = 0;
@ -900,7 +900,7 @@ static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq,
vInfo("vgId:%d, s3migrate vnode request will be processed, time:%d", pVnode->config.vgId, s3migrateReq.timestamp);
code = vnodeDoS3Migrate(pVnode, s3migrateReq.timestamp);
code = vnodeAsyncS3Migrate(pVnode, s3migrateReq.timestamp);
_exit:
return code;
@ -931,13 +931,13 @@ end:
return ret;
}
static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp) {
static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
int32_t code = -1;
SMetaReader mr = {0};
SVDropTtlTableReq ttlReq = {0};
SVFetchTtlExpiredTbsRsp rsp = {0};
SEncoder encoder = {0};
SArray* pNames = NULL;
SArray *pNames = NULL;
pRsp->msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP;
pRsp->code = TSDB_CODE_SUCCESS;
pRsp->pCont = NULL;
@ -950,8 +950,8 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void*
ASSERT(ttlReq.nUids == taosArrayGetSize(ttlReq.pTbUids));
tb_uid_t suid;
char ctbName[TSDB_TABLE_NAME_LEN];
tb_uid_t suid;
char ctbName[TSDB_TABLE_NAME_LEN];
SVDropTbReq expiredTb = {.igNotExists = true};
metaReaderDoInit(&mr, pVnode->pMeta, 0);
rsp.vgId = TD_VID(pVnode);
@ -965,12 +965,12 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void*
}
char buf[TSDB_TABLE_NAME_LEN];
for (int32_t i = 0; i < ttlReq.nUids; ++i) {
tb_uid_t* uid = taosArrayGet(ttlReq.pTbUids, i);
tb_uid_t *uid = taosArrayGet(ttlReq.pTbUids, i);
expiredTb.suid = *uid;
terrno = metaReaderGetTableEntryByUid(&mr, *uid);
if (terrno < 0) goto _end;
strncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN);
void* p = taosArrayPush(pNames, buf);
void *p = taosArrayPush(pNames, buf);
expiredTb.name = p;
if (mr.me.type == TSDB_CHILD_TABLE) {
expiredTb.suid = mr.me.ctbEntry.suid;
@ -1144,7 +1144,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
if (i < tbNames->size - 1) {
taosStringBuilderAppendChar(&sb, ',');
}
//taosMemoryFreeClear(*key);
// taosMemoryFreeClear(*key);
}
size_t len = 0;
@ -2241,14 +2241,14 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq,
return TSDB_CODE_SUCCESS;
}
extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
extern int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
if (!pVnode->restored) {
vInfo("vgId:%d, ignore compact req during restoring. ver:%" PRId64, TD_VID(pVnode), ver);
return 0;
}
return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp);
return vnodeAsyncCompact(pVnode, ver, pReq, len, pRsp);
}
static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
@ -2355,8 +2355,6 @@ _OVER:
}
#ifndef TD_ENTERPRISE
int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
return 0;
}
int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; }
int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; }
#endif

11
source/libs/executor/src/timesliceoperator.c
View File

@ -511,7 +511,6 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB
}
pInfo->isPrevRowSet = false;
return TSDB_CODE_SUCCESS;
}
@ -825,8 +824,12 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato
SSDataBlock* pResBlock = pSliceInfo->pRes;
SInterval* pInterval = &pSliceInfo->interval;
while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
pSliceInfo->fillType != TSDB_FILL_LINEAR) {
if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR ||
pSliceInfo->pPrevGroupKey == NULL) {
return;
}
while (pSliceInfo->current <= pSliceInfo->win.ekey) {
genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false);
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
@ -1068,6 +1071,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
// int32_t code = initKeeperInfo(pSliceInfo, pBlock, &pOperator->exprSupp);
code = appendDownstream(pOperator, &downstream, 1);
return pOperator;

2
source/libs/monitorfw/inc/taos_linked_list_i.h
View File

@ -47,7 +47,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item);
/**
* @brief API PRIVATE Pop the first item off of the list
*/
void *taos_linked_list_pop(taos_linked_list_t *self);
//void *taos_linked_list_pop(taos_linked_list_t *self);
/**
* @brief API PRIVATE Returns the item at the head of the list or NULL if not present

10
source/libs/monitorfw/src/taos_collector.c
View File

@ -33,6 +33,8 @@ taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self
taos_collector_t *taos_collector_new(const char *name) {
int r = 0;
taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t));
if (self == NULL) return NULL;
memset(self, 0, sizeof(taos_collector_t));
self->name = taos_strdup(name);
self->metrics = taos_map_new();
if (self->metrics == NULL) {
@ -66,9 +68,11 @@ int taos_collector_destroy(taos_collector_t *self) {
if (r) ret = r;
self->metrics = NULL;
r = taos_string_builder_destroy(self->string_builder);
if (r) ret = r;
self->string_builder = NULL;
if(self->string_builder != NULL){
r = taos_string_builder_destroy(self->string_builder);
if (r) ret = r;
self->string_builder = NULL;
}
taos_free((char *)self->name);
self->name = NULL;

4
source/libs/monitorfw/src/taos_collector_registry.c
View File

@ -59,6 +59,7 @@ taos_collector_registry_t *taos_collector_registry_new(const char *name) {
r = pthread_rwlock_init(self->lock, NULL);
if (r) {
TAOS_LOG("failed to initialize rwlock");
taos_free(self);
return NULL;
}
return self;
@ -301,6 +302,9 @@ const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self,
_OVER:
tjsonDelete(pJson);
if(tmp_builder != NULL){
taos_string_builder_destroy(tmp_builder);
}
return NULL;
}

2
source/libs/monitorfw/src/taos_linked_list.c
View File

@ -117,6 +117,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item) {
return 0;
}
/*
void *taos_linked_list_pop(taos_linked_list_t *self) {
TAOS_ASSERT(self != NULL);
if (self == NULL) return NULL;
@ -141,6 +142,7 @@ void *taos_linked_list_pop(taos_linked_list_t *self) {
}
return item;
}
*/
int taos_linked_list_remove(taos_linked_list_t *self, void *item) {
TAOS_ASSERT(self != NULL);

4
source/libs/monitorfw/src/taos_map.c
View File

@ -90,7 +90,7 @@ taos_map_t *taos_map_new() {
return NULL;
}
self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size);
self->addrs = taos_malloc(sizeof(taos_linked_list_t*) * self->max_size);
self->free_value_fn = destroy_map_node_value_no_op;
for (int i = 0; i < self->max_size; i++) {
@ -273,7 +273,7 @@ int taos_map_ensure_space(taos_map_t *self) {
if (r) return r;
// Create a new array of addrs
taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max);
taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t*) * new_max);
// Initialize the new array
for (int i = 0; i < new_max; i++) {

18
source/libs/monitorfw/src/taos_metric.c
View File

@ -33,6 +33,8 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name,
size_t label_key_count, const char **label_keys) {
int r = 0;
taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t));
if (self == NULL) return NULL;
memset(self, 0, sizeof(taos_metric_t));
self->type = metric_type;
int len = strlen(name) + 1;
self->name = taos_malloc(len);
@ -79,6 +81,7 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name,
r = pthread_rwlock_init(self->rwlock, NULL);
if (r) {
TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR);
taos_free(self);
return NULL;
}
return self;
@ -91,9 +94,11 @@ int taos_metric_destroy(taos_metric_t *self) {
int r = 0;
int ret = 0;
r = taos_map_destroy(self->samples);
self->samples = NULL;
if (r) ret = r;
if(self->samples != NULL){
r = taos_map_destroy(self->samples);
self->samples = NULL;
if (r) ret = r;
}
r = taos_metric_formatter_destroy(self->formatter);
self->formatter = NULL;
@ -152,8 +157,7 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const
return NULL;
// Get l_value
r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys,
label_values);
r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, label_values);
if (r) {
TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK();
}
@ -170,10 +174,12 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const
sample = taos_metric_sample_new(self->type, l_value, 0.0);
r = taos_map_set(self->samples, l_value, sample);
if (r) {
taos_free((void *)l_value);
TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK();
}
}
pthread_rwlock_unlock(self->rwlock);
r = pthread_rwlock_unlock(self->rwlock);
if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR);
taos_free((void *)l_value);
return sample;
}

3
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -845,6 +845,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
COPY_SCALAR_FIELD(precision);
COPY_SCALAR_FIELD(isEmptyResult);
COPY_SCALAR_FIELD(timeLineResMode);
COPY_SCALAR_FIELD(timeLineFromOrderBy);
COPY_SCALAR_FIELD(timeLineCurMode);
COPY_SCALAR_FIELD(hasAggFuncs);
COPY_SCALAR_FIELD(hasRepeatScanFuncs);
@ -862,6 +863,8 @@ static int32_t setOperatorCopy(const SSetOperator* pSrc, SSetOperator* pDst) {
COPY_CHAR_ARRAY_FIELD(stmtName);
COPY_SCALAR_FIELD(precision);
COPY_SCALAR_FIELD(timeLineResMode);
COPY_SCALAR_FIELD(timeLineFromOrderBy);
return TSDB_CODE_SUCCESS;
}

102
source/libs/parser/src/parTranslater.c
View File

@ -959,7 +959,8 @@ static bool isTimeLineQuery(SNode* pStmt) {
return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineCurMode) ||
(TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineCurMode);
} else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) {
return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode;
return (TIME_LINE_MULTI == ((SSetOperator*)pStmt)->timeLineResMode) ||
(TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode);
} else {
return false;
}
@ -1000,18 +1001,64 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) {
return false;
}
SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum) {
SNodeList* pPartitionList = NULL;
SNode* pNode = NULL;
if (pOrderList->length <= nodesNum) {
return NULL;
}
pNode = nodesListGetNode(pOrderList, nodesNum);
SOrderByExprNode* pOrder = (SOrderByExprNode*)pNode;
if (!isPrimaryKeyImpl(pOrder->pExpr)) {
return NULL;
}
for (int32_t i = 0; i < nodesNum; ++i) {
pNode = nodesListGetNode(pOrderList, i);
pOrder = (SOrderByExprNode*)pNode;
nodesListMakeStrictAppend(&pPartitionList, nodesCloneNode(pOrder->pExpr));
}
return pPartitionList;
}
static bool isTimeLineAlignedQuery(SNode* pStmt) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt;
if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
return false;
}
if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
return false;
if (QUERY_NODE_SELECT_STMT == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
if (pSelect->pPartitionByList) {
if (!pSub->timeLineFromOrderBy && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) {
return true;
}
if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
nodesDestroyList(pPartitionList);
if (match) {
return true;
}
}
}
}
SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
if (pSelect->pPartitionByList && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) {
return true;
if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
nodesDestroyList(pPartitionList);
if (match) {
return true;
}
}
}
return false;
}
@ -6025,9 +6072,19 @@ static void resetResultTimeline(SSelectStmt* pSelect) {
if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder)) ||
(QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) {
pSelect->timeLineResMode = TIME_LINE_GLOBAL;
} else {
pSelect->timeLineResMode = TIME_LINE_NONE;
return;
} else if (pSelect->pOrderByList->length > 1) {
for (int32_t i = 1; i < pSelect->pOrderByList->length; ++i) {
pOrder = ((SOrderByExprNode*)nodesListGetNode(pSelect->pOrderByList, i))->pExpr;
if (isPrimaryKeyImpl(pOrder)) {
pSelect->timeLineResMode = TIME_LINE_MULTI;
pSelect->timeLineFromOrderBy = true;
return;
}
}
}
pSelect->timeLineResMode = TIME_LINE_NONE;
}
static int32_t replaceOrderByAliasForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
@ -6180,16 +6237,13 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS
}
snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName);
SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft);
if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight)) {
SColumnNode* pLCol = (SColumnNode*)pLeft;
SColumnNode* pRCol = (SColumnNode*)pRight;
bool isLeftPrimTs = isPrimaryKeyImpl(pLeft);
bool isRightPrimTs = isPrimaryKeyImpl(pRight);
if (isLeftPrimTs && isRightPrimTs) {
SColumnNode* pFCol = (SColumnNode*)pProj;
if (pLCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID && pRCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
if (pLCol->isPrimTs && pRCol->isPrimTs) {
pFCol->isPrimTs = true;
}
}
pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
pFCol->isPrimTs = true;
}
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -6225,9 +6279,19 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS
SNode* pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, 0))->pExpr;
if (isPrimaryKeyImpl(pOrder)) {
pSetOperator->timeLineResMode = TIME_LINE_GLOBAL;
} else {
pSetOperator->timeLineResMode = TIME_LINE_NONE;
return code;
} else if (pSetOperator->pOrderByList->length > 1) {
for (int32_t i = 1; i < pSetOperator->pOrderByList->length; ++i) {
pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, i))->pExpr;
if (isPrimaryKeyImpl(pOrder)) {
pSetOperator->timeLineResMode = TIME_LINE_MULTI;
pSetOperator->timeLineFromOrderBy = true;
return code;
}
}
}
pSetOperator->timeLineResMode = TIME_LINE_NONE;
}
return code;
}

1
source/libs/planner/src/planLogicCreater.c
View File

@ -1587,6 +1587,7 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator
TSWAP(pProject->node.pLimit, pSetOperator->pLimit);
}
pProject->ignoreGroupId = true;
pProject->isSetOpProj = true;
int32_t code = TSDB_CODE_SUCCESS;

30
source/libs/planner/src/planOptimizer.c
View File

@ -3284,18 +3284,34 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
SNodeList* pNewChildTargets = nodesMakeList();
if (NULL == pProjectNode->node.pParent) {
SNode* pProjection = NULL;
FOREACH(pProjection, pProjectNode->pProjections) {
SNode* pChildTarget = NULL;
FOREACH(pChildTarget, pChild->pTargets) {
if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) {
nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget));
SNode *pProjection = NULL, *pChildTarget = NULL;
bool needOrderMatch = QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj;
bool orderMatch = true;
if (needOrderMatch) {
// For sql: select ... from (select ... union all select ...);
// When eliminating the outer proj (the outer select), we have to make sure that the outer proj projections and
// union all project targets have same columns in the same order. See detail in TD-30188
FORBOTH(pProjection, pProjectNode->pProjections, pChildTarget, pChild->pTargets) {
if (!pProjection) break;
if (0 != strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) {
orderMatch = false;
break;
}
nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget));
}
} else {
FOREACH(pProjection, pProjectNode->pProjections) {
FOREACH(pChildTarget, pChild->pTargets) {
if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) {
nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget));
break;
}
}
}
}
if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets)) {
if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets) &&
(!needOrderMatch || (needOrderMatch && orderMatch))) {
nodesDestroyList(pChild->pTargets);
pChild->pTargets = pNewChildTargets;
} else {

4
source/libs/qworker/src/qwMsg.c
View File

@ -370,12 +370,12 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran
if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) {
QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask);
tFreeSSubQueryMsg(&msg);
QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED);
QW_ERR_RET(TSDB_CODE_GRANT_VIEW_EXPIRED);
}
if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) {
QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask);
tFreeSSubQueryMsg(&msg);
QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED);
QW_ERR_RET(TSDB_CODE_GRANT_AUDIT_EXPIRED);
}
}
}

7
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -798,6 +798,7 @@ TEST(queryTest, normalCase) {
schedulerFreeJob(&job, 0);
taosThreadJoin(thread1, NULL);
}
TEST(queryTest, readyFirstCase) {
@ -907,6 +908,8 @@ TEST(queryTest, readyFirstCase) {
schedulerDestroy();
schedulerFreeJob(&job, 0);
taosThreadJoin(thread1, NULL);
}
TEST(queryTest, flowCtrlCase) {
@ -1001,6 +1004,8 @@ TEST(queryTest, flowCtrlCase) {
schedulerDestroy();
schedulerFreeJob(&job, 0);
taosThreadJoin(thread1, NULL);
}
TEST(insertTest, normalCase) {
@ -1061,6 +1066,8 @@ TEST(insertTest, normalCase) {
schedulerFreeJob(&insertJobRefId, 0);
schedulerDestroy();
taosThreadJoin(thread1, NULL);
}
TEST(multiThread, forceFree) {

17
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -1155,10 +1155,13 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
taskDbAddRef(pTaskDb);
int64_t chkpId = pTaskDb->chkpId;
taskDbRefChkp(pTaskDb, chkpId);
code = taskDbDoCheckpoint(pTaskDb, chkpId);
taskDbRemoveRef(pTaskDb);
if (code != 0) {
taskDbUnRefChkp(pTaskDb, chkpId);
}
taskDbRefChkp(pTaskDb, pTaskDb->chkpId);
taskDbRemoveRef(pTaskDb);
SStreamTask* pTask = pTaskDb->pTask;
SStreamTaskSnap snap = {.streamId = pTask->id.streamId,
@ -1182,14 +1185,15 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) {
for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) {
SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfo, i);
sprintf(buf, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)pSnap->taskId);
STaskDbWrapper* pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf));
if (pTaskDb == NULL) {
STaskDbWrapper** pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf));
if (pTaskDb == NULL || *pTaskDb == NULL) {
stWarn("stream backend:%p failed to find task db, streamId:% " PRId64 "", pMeta, pSnap->streamId);
memset(buf, 0, sizeof(buf));
continue;
}
memset(buf, 0, sizeof(buf));
taskDbUnRefChkp(pTaskDb, pSnap->chkpId);
taskDbUnRefChkp(*pTaskDb, pSnap->chkpId);
}
taosThreadMutexUnlock(&pMeta->backendMutex);
return 0;
@ -1989,7 +1993,8 @@ void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) {
void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) {
taosThreadRwlockWrlock(&pTaskDb->chkpDirLock);
for (int i = 0; i < taosArrayGetSize(pTaskDb->chkpInUse); i++) {
int32_t size = taosArrayGetSize(pTaskDb->chkpInUse);
for (int i = 0; i < size; i++) {
int64_t* p = taosArrayGet(pTaskDb->chkpInUse, i);
if (*p == chkp) {
taosArrayRemove(pTaskDb->chkpInUse, i);

31
source/libs/stream/src/streamExec.c
View File

@ -184,16 +184,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i
return code;
}
static int32_t handleResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) {
static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) {
int32_t code = TSDB_CODE_SUCCESS;
if (taosArrayGetSize(pRes) > 0) {
SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes);
code = doOutputResultBlockImpl(pTask, pStreamBlocks);
if (code != TSDB_CODE_SUCCESS) {
stDebug("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code));
if (code != TSDB_CODE_SUCCESS) { // should not have error code
stError("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code));
}
} else {
taosArrayDestroy(pRes);
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
}
return code;
}
@ -268,6 +268,17 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0);
}
// output queue is full, idle for 5 sec.
if (streamQueueIsFull(pTask->outputq.queue)) {
stWarn("s-task:%s outputQ is full, idle for 1sec and retry", id);
return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, STREAM_SCAN_HISTORY_TIMESLICE);
}
if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
stWarn("s-task:%s downstream task inputQ blocked, idle for 5sec and retry", id);
return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, FILL_HISTORY_TASK_EXEC_INTERVAL);
}
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
if (pRes == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -284,19 +295,13 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
}
// dispatch the generated results
/*int32_t code = */handleResultBlocks(pTask, pRes, size);
int64_t el = taosGetTimestampMs() - st;
// downstream task input queue is full, try in 5sec
if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED && (pTask->info.fillHistory == 1)) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, FILL_HISTORY_TASK_EXEC_INTERVAL);
}
/*int32_t code = */handleSanhistoryResultBlocks(pTask, pRes, size);
if (finished) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0);
}
int64_t el = taosGetTimestampMs() - st;
if (el >= STREAM_SCAN_HISTORY_TIMESLICE && (pTask->info.fillHistory == 1)) {
stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", id,
pTask->info.fillHistory, el / 1000.0);
@ -558,7 +563,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
if (streamQueueIsFull(pTask->outputq.queue)) {
stWarn("s-task:%s outputQ is full, idle for 500ms and retry", id);
streamTaskSetIdleInfo(pTask, 500);
streamTaskSetIdleInfo(pTask, 1000);
return 0;
}

11
source/libs/stream/src/streamMeta.c
View File

@ -562,13 +562,16 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
tEncoderClear(&encoder);
int64_t id[2] = {pTask->id.streamId, pTask->id.taskId};
if (tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn) < 0) {
stError("s-task:%s save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno));
return -1;
code = tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn);
if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s task meta save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno));
} else {
stDebug("s-task:%s task meta save to disk", pTask->id.idStr);
}
taosMemoryFree(buf);
return 0;
return code;
}
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pTaskId) {

2
source/libs/stream/src/streamQueue.c
View File

@ -381,7 +381,7 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc
}
}
return TSDB_CODE_SUCCESS;
return code;
}
int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate,

4
source/libs/stream/src/streamSnapshot.c
View File

@ -144,7 +144,7 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) {
int32_t streamCreateTaskDbSnapInfo(void* arg, char* path, SArray* pSnap) { return taskDbBuildSnap(arg, pSnap); }
int32_t streamDestroyTasdDbSnapInfo(void* arg, SArray* snap) { return taskDbDestroySnap(arg, snap); }
int32_t streamDestroyTaskDbSnapInfo(void* arg, SArray* snap) { return taskDbDestroySnap(arg, snap); }
void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) {
if (qDebugFlag & DEBUG_DEBUG) {
@ -333,7 +333,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
}
taosArrayDestroy(handle->pDbSnapSet);
}
streamDestroyTasdDbSnapInfo(handle->pMeta, handle->pSnapInfoSet);
streamDestroyTaskDbSnapInfo(handle->pMeta, handle->pSnapInfoSet);
if (handle->pSnapInfoSet) {
for (int32_t i = 0; i < taosArrayGetSize(handle->pSnapInfoSet); i++) {
SStreamTaskSnap* pSnap = taosArrayGet(handle->pSnapInfoSet, i);

2
source/libs/stream/src/tstreamFileState.c
View File

@ -759,7 +759,7 @@ void* getRowStateBuff(SStreamFileState* pFileState) { return pFileState->rowStat
void* getStateFileStore(SStreamFileState* pFileState) { return pFileState->pFileStore; }
bool isDeteled(SStreamFileState* pFileState, TSKEY ts) {
return pFileState->deleteMark > 0 && ts < (pFileState->maxTs - pFileState->deleteMark);
return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 && ts < (pFileState->maxTs - pFileState->deleteMark);
}
bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return ts <= (pFileState->flushMark + gap); }

5
source/libs/tfs/src/tfs.c
View File

@ -177,11 +177,10 @@ int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) {
continue;
}
return 0;
return (terrno = 0);
}
terrno = TSDB_CODE_FS_NO_VALID_DISK;
return -1;
return (terrno = TSDB_CODE_FS_NO_VALID_DISK);
}
const char *tfsGetPrimaryPath(STfs *pTfs) { return TFS_PRIMARY_DISK(pTfs)->path; }

4
source/util/src/tcompression.c
View File

@ -2741,7 +2741,7 @@ int32_t tsDecompressTimestamp2(void *pIn, int32_t nIn, int32_t nEle, void *pOut,
int32_t tsCompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf) {
DEFINE_VAR(cmprAlg)
if (lvl != 0 && lossyFloat) {
if (l2 == L2_TSZ && lvl != 0 && lossyFloat) {
return tsCompressFloatLossyImp(pIn, nEle, pOut);
}
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_FLOAT, 1);
@ -2760,7 +2760,7 @@ int32_t tsDecompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int
int32_t tsCompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf) {
DEFINE_VAR(cmprAlg)
if (lvl != 0 && lossyDouble) {
if (l2 == L2_TSZ && lvl != 0 && lossyDouble) {
// lossy mode
return tsCompressDoubleLossyImp(pIn, nEle, pOut);
}

16
source/util/src/terror.c
View File

@ -259,7 +259,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY, "Inconsistent encryption key")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ENCRYPT_KEY, "The cluster has not been set properly for database encryption")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED, "The database encryption function grant expired")
// mnode-node
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists")
@ -499,9 +498,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LACK_OF_BASIC, "Lack of basic functio
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJ_NOT_EXIST, "Grant object not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LAST_ACTIVE_NOT_FOUND, "The historial active code does not match")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MACHINES_MISMATCH, "Cluster machines mismatch with active code")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expire time of optional grant item is too large")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expiration time of optional grant item is too large")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUPLICATED_ACTIVE, "The active code can't be activated repeatedly")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of view has reached the licensed upper limit")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of views has reached the licensed upper limit")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_BASIC_EXPIRED, "License expired for basic functions")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_EXPIRED, "License expired for stream function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED, "License expired for subscription function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_EXPIRED, "License expired for view function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_AUDIT_EXPIRED, "License expired for audit function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CSV_EXPIRED, "License expired for CSV function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED, "License expired for multi-tier storage function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED, "License expired for object storage function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED,"License expired for dual-replica HA function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED, "License expired for database encryption function")
// sync
TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout")

11
tests/army/community/insert/test_column_tag_boundary.py
View File

@ -22,6 +22,7 @@ class TDTestCase(TBase):
self.child_table_num = 1
self.insert_round_num = 700
self.row_num_per_round = 15
self.start_ts = 1704082431000
def prepare_data(self):
# database
@ -39,7 +40,7 @@ class TDTestCase(TBase):
for j in range(self.insert_round_num):
sql = "insert into ct_binary%s values" % (i+1)
for k in range(self.row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'a' * self.max_column_length)
sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), 'a' * self.max_column_length)
tdSql.execute(sql)
tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_binary{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")
@ -63,7 +64,7 @@ class TDTestCase(TBase):
for j in range(self.insert_round_num):
sql = "insert into ct_varchar%s values" % (i+1)
for k in range(self.row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'b' * self.max_column_length)
sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), 'b' * self.max_column_length)
tdSql.execute(sql)
tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_varchar{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")
@ -98,7 +99,7 @@ class TDTestCase(TBase):
for j in range(self.insert_round_num):
sql = "insert into ct_nchar%s values" % (i+1)
for k in range(self.row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), column)
sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), column)
tdSql.execute(sql)
tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_nchar{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")
@ -124,7 +125,7 @@ class TDTestCase(TBase):
for j in range(self.insert_round_num):
sql = "insert into ct_varbinary%s values" % (i+1)
for k in range(row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column)
sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), '\\x' + column)
tdSql.execute(sql)
tdLog.info(f"Insert {row_num_per_round} rows data into ct_varbinary{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")
@ -153,7 +154,7 @@ class TDTestCase(TBase):
for j in range(self.insert_round_num):
sql = "insert into ct_json_tag%s values" % (i+1)
for k in range(row_num_per_round):
sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column)
sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), '\\x' + column)
tdSql.execute(sql)
tdLog.info(f"Insert {row_num_per_round} rows data into ct_json_tag{i+1} {j+1} times successfully")
tdSql.execute("flush database db;")

77
tests/army/community/query/accuracy/test_query_accuracy.py Normal file
View File

@ -0,0 +1,77 @@
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
from frame.eos import *
import random
import string
class TDTestCase(TBase):
"""Add test case to verify the complicated query accuracy
"""
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
def prepare_data(self):
# database for case TS-4806
tdSql.execute("create database db_ts4806;")
tdSql.execute("use db_ts4806;")
# super table
tdSql.execute("create table st (ts timestamp, adl float, bdl float, cdl float, ady float, bdy float, cdy float) \
tags(pt_radio float, ct_ratio float, rated_cap float, ta_id varchar(128), id varchar(128), area_code \
varchar(128), zdy_flag int, elec_cust_name bigint,bureau_code bigint, fl_name varchar(32), classify_id \
varchar(128));")
# child table
tdSql.execute("create table ct_1 using st tags(1.2, 1.3, 3.4, '271000276', '30000001', '10001', 1, 10001, 2000001, 'beijing', '13169');")
tdSql.execute("create table ct_2 using st tags(2.1, 1.2, 3.3, '271000277', '30000002', '10002', 1, 10002, 2000002, 'shanghai', '13141');")
tdSql.execute("create table ct_3 using st tags(3.1, 4.2, 5.3, '271000278', '30000003', '10003', 0, 10003, 2000003, 'guangzhou', '13151');")
# insert data for ts4806
start_ts = 1705783972000
data = [
(1.1, 2.2, 3.3, 1.1, 2.2, 3.3),
(1.2, 2.3, 3.4, 1.2, 2.3, 3.4),
(1.3, 2.4, 3.5, 1.3, 2.4, 3.5),
(1.4, 2.5, 3.6, 1.4, 2.5, 3.6),
(1.5, 2.6, 3.7, 1.5, 2.6, 3.7),
(1.6, 2.7, 3.8, 1.6, 2.7, 3.8),
(1.7, 2.8, 3.9, 1.7, 2.8, 3.9),
(1.8, 2.9, 4.0, 1.8, 2.9, 4.0),
(1.9, 4.2, 4.1, 1.9, 3.0, 4.1),
(1.2, 3.1, 4.2, 2.0, 3.1, 4.2)
]
index = [1, 2, 5, 0, 7, 3, 8, 4, 6, 9]
for ct in ['ct_1', 'ct_2']:
for i in range(10):
sql = f"insert into {ct} values"
for j in range(1000):
sql += f"({start_ts + i * 1000 * 1000 + j * 1000}, {','.join([str(item) for item in data[index[i]]])}),"
sql += ";"
tdSql.execute(sql)
def test_ts4806(self):
tdSql.execute("use db_ts4806;")
tdSql.query("select _wstart, cj.id, count(*) from st cj where cj.ts >= '2024-01-21 04:52:52.000' and cj.ts <= ' 2024-01-21 07:39:31.000' \
and cj.zdy_flag = 1 and cj.id in ('30000001', '30000002') partition by cj.id event_window start with \
(CASE WHEN cj.adl >= cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl \
THEN cj.bdl ELSE cj.cdl END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap > 1 end with (CASE WHEN cj.adl >= \
cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl THEN cj.bdl ELSE cj.cdl \
END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap <= 1 HAVING count(*) >= 4 order by _wstart, cj.id;")
tdSql.checkRows(5)
tdSql.checkData(4, 1, '30000002')
tdSql.checkData(4, 2, 1001)
def run(self):
self.prepare_data()
self.test_ts4806()
def stop(self):
tdSql.execute("drop database db_ts4806;")
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

55
tests/army/enterprise/alter/alterConfig.py Normal file
View File

@ -0,0 +1,55 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
import time
import taos
import frame
import frame.etool
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.caseBase import *
from frame import *
class TDTestCase(TBase):
def alterSupportVnodes(self):
tdLog.info(f"test function of altering supportVnodes")
tdSql.execute("alter dnode 1 'supportVnodes' '128'")
time.sleep(1)
tdSql.query('show dnodes')
tdSql.checkData(0, 3, "128")
tdSql.execute("alter dnode 1 'supportVnodes' '64'")
time.sleep(1)
tdSql.query('show dnodes')
tdSql.checkData(0, 3, "64")
# run
def run(self):
tdLog.debug(f"start to excute {__file__}")
# TS-4721
self.alterSupportVnodes()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

63
tests/army/enterprise/db-encrypt/basic.py Normal file
View File

@ -0,0 +1,63 @@
import taos
import sys
import os
import subprocess
import glob
import shutil
import time
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.srvCtl import *
from frame.caseBase import *
from frame import *
from frame.autogen import *
# from frame.server.dnodes import *
# from frame.server.cluster import *
class TDTestCase(TBase):
def init(self, conn, logSql, replicaVar=1):
super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1")
self.valgrind = 0
self.db = "test"
self.stb = "meters"
self.childtable_count = 10
tdSql.init(conn.cursor(), logSql)
def create_encrypt_db(self):
tdSql.execute("create encrypt_key '1234567890'")
autoGen = AutoGen()
autoGen.create_db(self.db, 2, 1, "ENCRYPT_ALGORITHM 'sm4'")
tdSql.execute(f"use {self.db}")
autoGen.create_stable(self.stb, 2, 3, 8, 8)
autoGen.create_child(self.stb, "d", self.childtable_count)
autoGen.insert_data(1000)
tdSql.query(f"select * from {self.db}.{self.stb}")
tdSql.checkRows(1000 * self.childtable_count)
self.timestamp_step = 1000
self.insert_rows = 1000
self.checkInsertCorrect()
def create_encrypt_db_error(self):
tdSql.error("create encrypt_key '123'")
tdSql.error("create encrypt_key '12345678abcdefghi'")
tdSql.error("create database test ENCRYPT_ALGORITHM 'sm4'")
def run(self):
self.create_encrypt_db_error()
self.create_encrypt_db()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

4
tests/parallel_test/cases.task
View File

@ -11,6 +11,7 @@
# army-test
#
,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f enterprise/db-encrypt/basic.py
,,n,army,python3 ./test.py -f enterprise/s3/s3Basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py
@ -20,10 +21,12 @@
,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/query/accuracy/test_query_accuracy.py
,,y,army,./pytest.sh python3 ./test.py -f community/insert/insert_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
,,n,army,python3 ./test.py -f community/query/show.py -N 3
,,n,army,python3 ./test.py -f enterprise/alter/alterConfig.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3
@ -994,6 +997,7 @@
,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py
#tsim test
,,y,script,./test.sh -f tsim/query/timeline.sim
,,y,script,./test.sh -f tsim/join/join.sim
,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim
,,y,script,./test.sh -f tsim/parser/where.sim

27
tests/parallel_test/run.sh
View File

@ -315,19 +315,22 @@ function run_thread() {
fi
if [ -n "$corefile" ]; then
echo -e "\e[34m corefiles: $corefile \e[0m"
local build_dir=$log_dir/build_${hosts[index]}
local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build"
# if [ $ent -ne 0 ]; then
# remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build"
# fi
mkdir "$build_dir" 2>/dev/null
if [ $? -eq 0 ]; then
# scp build binary
cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/"
echo "$cmd"
$cmd >/dev/null
fi
fi
# scp build binary and unit test log
local build_dir=$log_dir/build_${hosts[index]}
local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build"
local remote_unit_test_log_dir="${workdirs[index]}/${DEBUGPATH}/Testing/Temporary/"
mkdir "$build_dir" 2>/dev/null
if [ $? -eq 0 ]; then
cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/"
echo "$cmd"
$cmd >/dev/null
cmd="$scpcmd:${remote_unit_test_log_dir}/* ${build_dir}/"
echo "$cmd"
$cmd >/dev/null
fi
# get remote sim dir
local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no"
local tarcmd="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}"

48
tests/pytest/util/common.py
View File

@ -201,6 +201,9 @@ class TDCom:
self.cast_tag_stb_filter_des_select_elm = "ts, t1, t2, t3, t4, cast(t1 as TINYINT UNSIGNED), t6, t7, t8, t9, t10, cast(t2 as varchar(256)), t12, cast(t3 as bool)"
self.tag_count = len(self.tag_filter_des_select_elm.split(","))
self.state_window_range = list()
self.custom_col_val = 0
self.part_val_list = [1, 2]
# def init(self, conn, logSql):
# # tdSql.init(conn.cursor(), logSql)
@ -1259,7 +1262,7 @@ class TDCom:
default_ctbname_index_start_num += 1
tdSql.execute(create_stable_sql)
def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None):
def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None, additional_ts=None, custom_col_index=None, col_value_type=None, force_pk_val=None):
"""_summary_
Args:
@ -1269,6 +1272,8 @@ class TDCom:
"""
self.column_value_list = list()
self.ts_value = self.genTs()[0]
if additional_ts is not None:
self.additional_ts = self.genTs(additional_ts=additional_ts)[2]
if ts_value is not None:
self.ts_value = ts_value
@ -1292,7 +1297,22 @@ class TDCom:
for i in range(int(len(self.column_value_list)/2)):
index_num = random.randint(0, len(self.column_value_list)-1)
self.column_value_list[index_num] = None
self.column_value_list = [self.ts_value] + self.column_value_list
if custom_col_index is not None:
if col_value_type == "Random":
pass
elif col_value_type == "Incremental":
self.column_value_list[custom_col_index] = self.custom_col_val
self.custom_col_val += 1
elif col_value_type == "Part_equal":
self.column_value_list[custom_col_index] = random.choice(self.part_val_list)
self.column_value_list = [self.ts_value] + [self.additional_ts] + self.column_value_list if additional_ts is not None else [self.ts_value] + self.column_value_list
if col_value_type == "Incremental" and custom_col_index==1:
self.column_value_list[custom_col_index] = self.custom_col_val if force_pk_val is None else force_pk_val
if col_value_type == "Part_equal" and custom_col_index==1:
self.column_value_list[custom_col_index] = random.randint(0, self.custom_col_val) if force_pk_val is None else force_pk_val
def screate_table(self, dbname=None, tbname="tb", use_name="table", column_elm_list=None,
count=1, default_tbname_prefix="tb", default_tbname_index_start_num=1,
@ -1333,7 +1353,7 @@ class TDCom:
default_tbname_index_start_num += 1
tdSql.execute(create_table_sql)
def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False):
def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False, custom_col_index=None, col_value_type="random"):
"""insert rows
Args:
@ -1353,7 +1373,7 @@ class TDCom:
if tbname is not None:
self.tbname = tbname
self.sgen_column_value_list(column_ele_list, need_null, ts_value)
self.sgen_column_value_list(column_ele_list, need_null, ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
# column_value_str = ", ".join(str(v) for v in self.column_value_list)
column_value_str = ""
for column_value in self.column_value_list:
@ -1370,7 +1390,7 @@ class TDCom:
else:
for num in range(count):
ts_value = self.genTs()[0]
self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s')
self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s', custom_col_index=custom_col_index, col_value_type=col_value_type)
column_value_str = ""
for column_value in self.column_value_list:
if column_value is None:
@ -1777,7 +1797,7 @@ class TDCom:
self.sdelete_rows(tbname=self.ctb_name, start_ts=self.time_cast(self.record_history_ts, "-"))
self.sdelete_rows(tbname=self.tb_name, start_ts=self.time_cast(self.record_history_ts, "-"))
def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None):
def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None, custom_col_index=None, col_value_type="random"):
"""prepare stream data
Args:
@ -1840,8 +1860,8 @@ class TDCom:
if fill_history_value == 1:
for i in range(self.range_count):
ts_value = str(self.date_time)+f'-{self.default_interval*(i+1)}s'
self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value)
self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value)
self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
if i == 1:
self.record_history_ts = ts_value
@ -1862,6 +1882,18 @@ class TDCom:
time.sleep(1)
return tbname
def get_group_id_from_stb(self, stbname):
tdSql.query(f'select distinct group_id from {stbname}')
cnt = 0
while len(tdSql.queryResult) == 0:
tdSql.query(f'select distinct group_id from {stbname}')
if cnt < self.default_interval:
cnt += 1
time.sleep(1)
else:
return False
return tdSql.queryResult[0][0]
def update_json_file_replica(self, json_file_path, new_replica_value, output_file_path=None):
"""
Read a JSON file, update the 'replica' value, and write the result back to a file.

51
tests/script/tsim/query/timeline.sim Normal file
View File

@ -0,0 +1,51 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql create database test;
sql use test;
sql CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256));
sql CREATE TABLE demo_1 USING demo (deviceid) TAGS ('1');
sql CREATE TABLE demo_2 USING demo (deviceid) TAGS ('2');
sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 1.0);
sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 01:00:00.001', 2.0);
sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 02:00:00.002', 3.0);
sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-05 03:00:00.003', 4.0);
sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 5.0);
sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 01:00:00.001', 6.0);
sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 02:00:00.002', 7.0);
sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-29 03:00:00.003', 8.0);
sql_error select diff(faev) from ((select ts, faev from demo union all select ts, faev from demo));
sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts);
sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts) partition by faev;
sql select diff(faev) from (select _ts, faev from demo union all select _ts + 1s, faev from demo order by faev, _ts) partition by faev;
sql_error select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by deviceid, _ts) partition by faev;
sql select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev;
sql_error select diff(faev) from (select _ts, faev from demo);
sql_error select diff(faev) from (select _ts, faev from demo order by faev, _ts);
sql select diff(faev) from (select _ts, faev from demo order by faev, _ts) partition by faev;
sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts) partition by deviceid;
sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by deviceid, _ts) partition by faev;
sql select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev;
sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY deviceid, ts) PARTITION by deviceid;
sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY ts, deviceid) PARTITION by deviceid;
sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid;
sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid;
sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid;
sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid;
sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY deviceid, ts) PARTITION by deviceid;
sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY ts, deviceid) PARTITION by deviceid;
system sh/exec.sh -n dnode1 -s stop -x SIGINT

8
tests/system-test/0-others/information_schema.py
View File

@ -222,7 +222,7 @@ class TDTestCase:
tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
tdLog.info(len(tdSql.queryResult))
tdSql.checkEqual(True, len(tdSql.queryResult) in range(254, 255))
tdSql.checkEqual(True, len(tdSql.queryResult) in range(255, 256))
tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
tdSql.checkEqual(54, len(tdSql.queryResult))
@ -336,7 +336,7 @@ class TDTestCase:
tdSql.checkEqual(True, result[i][1] in key_status_list[1])
index += 1
tdSql.checkEqual(True, index > 0)
tdSql.query(f'show encryptions')
result = tdSql.queryResult
index = 0
@ -344,7 +344,7 @@ class TDTestCase:
tdSql.checkEqual(True, result[i][1] in key_status_list[1])
index += 1
tdSql.checkEqual(True, index > 0)
# loaded/sm4
tdSql.execute('drop database if exists db2')
tdSql.execute('create encrypt_key \'12345678\'')
@ -357,7 +357,7 @@ class TDTestCase:
tdSql.checkEqual(True, result[i][1] in key_status_list[3])
index += 1
tdSql.checkEqual(True, index > 0)
tdSql.query(f'show encryptions')
result = tdSql.queryResult
index = 0

25
tests/system-test/2-query/interp.py
View File

@ -15,6 +15,30 @@ class TDTestCase:
#tdSql.init(conn.cursor())
tdSql.init(conn.cursor(), logSql) # output sql.txt file
def interp_on_empty_table(self):
dbname = "db"
tbname = "t"
tdSql.prepare()
tdLog.printNoPrefix("==========step1:create table")
tdSql.execute(f'''create table if not exists {dbname}.{tbname} (ts timestamp, k int)''')
tdLog.printNoPrefix("==========step2:interp query on empty table")
tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(prev)")
tdSql.checkRows(0)
tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(next)")
tdSql.checkRows(0)
tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(linear)")
tdSql.checkRows(0)
tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(value, 2)")
tdSql.checkRows(0)
def run(self):
dbname = "db"
tbname = "tb"
@ -5658,6 +5682,7 @@ class TDTestCase:
tdSql.checkData(0, 0, '2023-08-06 23:59:00')
tdSql.checkData(0, 1, None)
self.interp_on_empty_table()
def stop(self):

13
tests/system-test/2-query/union1.py
View File

@ -240,9 +240,22 @@ class TDTestCase:
tdSql.error( " '' union all select c1 from ct1 " )
# tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
def test_select_from_union_all(self):
tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all select ts, c8, c1 from stb1 limit 15)')
tdSql.checkRows(15)
tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
tdSql.checkRows(15)
tdSql.query('select ts, c1 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
tdSql.checkRows(15)
tdSql.query('select ts, c1, c8 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
tdSql.checkRows(15)
tdSql.query('select ts, c8, c1, 123 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
tdSql.checkRows(15)
def all_test(self):
self.__test_error()
self.union_check()
self.test_select_from_union_all()
def __create_tb(self):

20
tests/system-test/8-stream/at_once_interval.py
View File

@ -15,9 +15,12 @@ class TDTestCase:
def at_once_interval(self, interval, partition="tbname", delete=False, fill_value=None, fill_history_value=None, case_when=None):
tdLog.info(f"*** testing stream at_once+interval: interval: {interval}, partition: {partition}, fill_history: {fill_history_value}, fill: {fill_value}, delete: {delete}, case_when: {case_when} ***")
col_value_type = "Incremental" if partition=="c1" else "random"
custom_col_index = 1 if partition=="c1" else None
self.tdCom.custom_col_val = 0
self.delete = delete
self.tdCom.case_name = sys._getframe().f_code.co_name
self.tdCom.prepare_data(interval=interval, fill_history_value=fill_history_value)
self.tdCom.prepare_data(interval=interval, fill_history_value=fill_history_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
self.stb_name = self.tdCom.stb_name.replace(f"{self.tdCom.dbname}.", "")
self.ctb_name = self.tdCom.ctb_name.replace(f"{self.tdCom.dbname}.", "")
self.tb_name = self.tdCom.tb_name.replace(f"{self.tdCom.dbname}.", "")
@ -76,15 +79,15 @@ class TDTestCase:
for i in range(self.tdCom.range_count):
ts_value = str(self.tdCom.date_time+self.tdCom.dataDict["interval"])+f'+{i*10}s'
ts_cast_delete_value = self.tdCom.time_cast(ts_value)
self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value)
self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
if i%2 == 0:
self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value)
self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
if self.delete and i%2 != 0:
self.tdCom.sdelete_rows(tbname=self.tdCom.ctb_name, start_ts=ts_cast_delete_value)
self.tdCom.date_time += 1
self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value)
self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
if i%2 == 0:
self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value)
self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
if self.delete and i%2 != 0:
self.tdCom.sdelete_rows(tbname=self.tdCom.tb_name, start_ts=ts_cast_delete_value)
self.tdCom.date_time += 1
@ -102,6 +105,7 @@ class TDTestCase:
if self.tdCom.subtable:
for tname in [self.stb_name, self.ctb_name]:
group_id = self.tdCom.get_group_id_from_stb(f'{tname}_output')
tdSql.query(f'select * from {self.ctb_name}')
ptn_counter = 0
for c1_value in tdSql.queryResult:
@ -116,11 +120,11 @@ class TDTestCase:
tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{abs_c1_value}{self.tdCom.subtable_suffix}')
tdSql.query(f'select count(*) from `{tbname}`')
elif partition == "tbname" and ptn_counter == 0:
tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}')
tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}_{tname}_output_{group_id}')
tdSql.query(f'select count(*) from `{tbname}`')
ptn_counter += 1
tdSql.checkEqual(tdSql.queryResult[0][0] > 0, True)
group_id = self.tdCom.get_group_id_from_stb(f'{self.tb_name}_output')
tdSql.query(f'select * from {self.tb_name}')
ptn_counter = 0
for c1_value in tdSql.queryResult:
@ -135,7 +139,7 @@ class TDTestCase:
tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{abs_c1_value}{self.tdCom.subtable_suffix}')
tdSql.query(f'select count(*) from `{tbname}`')
elif partition == "tbname" and ptn_counter == 0:
tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}')
tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}_{self.tb_name}_output_{group_id}')
tdSql.query(f'select count(*) from `{tbname}`')
ptn_counter += 1

20
tests/system-test/8-stream/at_once_session.py
View File

@ -15,9 +15,12 @@ class TDTestCase:
def at_once_session(self, session, ignore_expired=None, ignore_update=None, partition="tbname", delete=False, fill_history_value=None, case_when=None, subtable=True):
tdLog.info(f"*** testing stream at_once+interval: session: {session}, ignore_expired: {ignore_expired}, ignore_update: {ignore_update}, partition: {partition}, delete: {delete}, fill_history: {fill_history_value}, case_when: {case_when}, subtable: {subtable} ***")
col_value_type = "Incremental" if partition=="c1" else "random"
custom_col_index = 1 if partition=="c1" else None
self.tdCom.custom_col_val = 0
self.delete = delete
self.tdCom.case_name = sys._getframe().f_code.co_name
self.tdCom.prepare_data(session=session, fill_history_value=fill_history_value)
self.tdCom.prepare_data(session=session, fill_history_value=fill_history_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
self.stb_name = self.tdCom.stb_name.replace(f"{self.tdCom.dbname}.", "")
self.ctb_name = self.tdCom.ctb_name.replace(f"{self.tdCom.dbname}.", "")
self.tb_name = self.tdCom.tb_name.replace(f"{self.tdCom.dbname}.", "")
@ -79,11 +82,11 @@ class TDTestCase:
if i == 0:
record_window_close_ts = window_close_ts
for ts_value in [self.tdCom.date_time, window_close_ts]:
self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True)
self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True)
self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type)
self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type)
if self.tdCom.update and i%2 == 0:
self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True)
self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True)
self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type)
self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type)
if self.delete and i%2 != 0:
dt = f'cast({self.tdCom.date_time-1} as timestamp)'
self.tdCom.sdelete_rows(tbname=self.ctb_name, start_ts=dt)
@ -166,6 +169,7 @@ class TDTestCase:
self.tdCom.sdelete_rows(tbname=self.tb_name, start_ts=self.tdCom.time_cast(self.tdCom.record_history_ts, "-"))
if self.tdCom.subtable:
group_id = self.tdCom.get_group_id_from_stb(f'{self.ctb_name}_output')
tdSql.query(f'select * from {self.ctb_name}')
ptn_counter = 0
for c1_value in tdSql.queryResult:
@ -182,11 +186,11 @@ class TDTestCase:
tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{partition_elm_alias}{self.tdCom.subtable_suffix}')
tdSql.query(f'select count(*) from `{tbname}`')
elif partition == "tbname" and ptn_counter == 0:
tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}')
tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}_{self.ctb_name}_output_{group_id}')
tdSql.query(f'select count(*) from `{tbname}`')
ptn_counter += 1
tdSql.checkEqual(tdSql.queryResult[0][0] > 0, True) if subtable is not None else tdSql.checkEqual(tdSql.queryResult[0][0] >= 0, True)
group_id = self.tdCom.get_group_id_from_stb(f'{self.tb_name}_output')
tdSql.query(f'select * from {self.tb_name}')
ptn_counter = 0
for c1_value in tdSql.queryResult:
@ -203,7 +207,7 @@ class TDTestCase:
tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{partition_elm_alias}{self.tdCom.subtable_suffix}')
tdSql.query(f'select count(*) from `{tbname}`')
elif partition == "tbname" and ptn_counter == 0:
tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}')
tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}_{self.tb_name}_output_{group_id}')
tdSql.query(f'select count(*) from `{tbname}`')
ptn_counter += 1

4
tests/system-test/8-stream/stream_multi_agg.py
View File

@ -39,7 +39,7 @@ class TDTestCase:
os.system("nohup taosBenchmark -y -B 1 -t 40 -S 1000 -n 10 -i 1000 -v 5 > /dev/null 2>&1 &")
time.sleep(10)
tdSql.query("use test")
tdSql.execute("use test", queryTimes=100)
tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)")
tdLog.debug("========create stream and insert data ok========")
time.sleep(15)
@ -66,7 +66,7 @@ class TDTestCase:
os.system("taosBenchmark -d db -t 20 -v 6 -n 1000 -y > /dev/null 2>&1")
# create stream
tdSql.execute("use db")
tdSql.execute("use db", queryTimes=100)
tdSql.execute("create stream stream1 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True)
time.sleep(5)

4
tests/system-test/pytest.sh
View File

@ -89,8 +89,8 @@ else
export LD_PRELOAD="$(realpath "$(gcc -print-file-name=libasan.so)") $(realpath "$(gcc -print-file-name=libstdc++.so)")"
echo "Preload AsanSo:" $?
$* -a 2>$AsanFile
$* -a 2> $AsanFile
cat $AsanFile
unset LD_PRELOAD
for ((i = 1; i <= 20; i++)); do
AsanFileLen=$(cat $AsanFile | wc -l)

20
tools/CMakeLists.txt
View File

@ -140,8 +140,8 @@ ELSE ()
BUILD_COMMAND
COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client
COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib
COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
INSTALL_COMMAND
COMMAND cmake -E echo "Comparessing taosadapter.exe"
@ -167,8 +167,8 @@ ELSE ()
PATCH_COMMAND
COMMAND git clean -f -d
BUILD_COMMAND
COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
INSTALL_COMMAND
COMMAND cmake -E echo "Copy taosadapter"
COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin
@ -192,19 +192,19 @@ ELSE ()
PATCH_COMMAND
COMMAND git clean -f -d
BUILD_COMMAND
COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
# COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
INSTALL_COMMAND
COMMAND cmake -E echo "Comparessing taosadapter.exe"
COMMAND upx taosadapter || :
# COMMAND cmake -E echo "Comparessing taosadapter.exe"
# COMMAND upx taosadapter || :
COMMAND cmake -E echo "Copy taosadapter"
COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin
COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/
COMMAND cmake -E echo "Copy taosadapter.toml"
COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/
COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/
COMMAND cmake -E echo "Copy taosadapter-debug"
COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin
# COMMAND cmake -E echo "Copy taosadapter-debug"
# COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin
)
ENDIF ()
ENDIF ()

2
utils/test/c/write_raw_block_test.c
View File

@ -78,7 +78,7 @@ int buildStable(TAOS* pConn) {
}
taos_free_result(pRes);
pRes = taos_query(pConn, "insert into ntba values(now,'hello')");
pRes = taos_query(pConn, "insert into ntba values(now + 1s,'hello')");
if (taos_errno(pRes) != 0) {
printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes));
return -1;