xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into enh/TS-23926-3.0

This commit is contained in:
xjzhou 2024-06-26 14:21:15 +08:00
parent 1c8110e7a4 0fb4ecbdad
commit 1a32279f7f
51 changed files with 1070 additions and 951 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/libs/stream/streammsg.h
View File

@ -182,12 +182,12 @@ typedef struct SRetrieveChkptTriggerReq {
} SRetrieveChkptTriggerReq; } SRetrieveChkptTriggerReq;
typedef struct SCheckpointTriggerRsp { typedef struct SCheckpointTriggerRsp {
int64_t streamId; int64_t streamId;
int64_t checkpointId; int64_t checkpointId;
int32_t upstreamTaskId; int32_t upstreamTaskId;
int32_t taskId; int32_t taskId;
int32_t transId; int32_t transId;
int32_t rspCode; int32_t rspCode;
} SCheckpointTriggerRsp; } SCheckpointTriggerRsp;
typedef struct SCheckpointReport { typedef struct SCheckpointReport {

115
include/libs/stream/tstream.h
View File

@ -18,13 +18,13 @@
#include "os.h" #include "os.h"
#include "streamState.h" #include "streamState.h"
#include "streammsg.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tdbInt.h" #include "tdbInt.h"
#include "tmsg.h" #include "tmsg.h"
#include "tmsgcb.h" #include "tmsgcb.h"
#include "tqueue.h" #include "tqueue.h"
#include "ttimer.h" #include "ttimer.h"
#include "streammsg.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -265,14 +265,14 @@ typedef struct SStreamTaskId {
} SStreamTaskId; } SStreamTaskId;
typedef struct SCheckpointInfo { typedef struct SCheckpointInfo {
int64_t startTs; int64_t startTs;
int64_t checkpointId; // latest checkpoint id int64_t checkpointId; // latest checkpoint id
int64_t checkpointVer; // latest checkpoint offset in wal int64_t checkpointVer; // latest checkpoint offset in wal
int64_t checkpointTime; // latest checkpoint time int64_t checkpointTime; // latest checkpoint time
int64_t processedVer; int64_t processedVer;
int64_t nextProcessVer; // current offset in WAL, not serialize it int64_t nextProcessVer; // current offset in WAL, not serialize it
SActiveCheckpointInfo* pActiveInfo; SActiveCheckpointInfo* pActiveInfo;
int64_t msgVer; int64_t msgVer;
} SCheckpointInfo; } SCheckpointInfo;
typedef struct SStreamStatus { typedef struct SStreamStatus {
@ -301,7 +301,7 @@ typedef struct SSTaskBasicInfo {
int32_t selfChildId; int32_t selfChildId;
int32_t totalLevel; int32_t totalLevel;
int8_t taskLevel; int8_t taskLevel;
int8_t fillHistory; // is fill history task or not int8_t fillHistory; // is fill history task or not
int64_t delaySchedParam; // in msec int64_t delaySchedParam; // in msec
} SSTaskBasicInfo; } SSTaskBasicInfo;
@ -323,7 +323,7 @@ typedef struct SDispatchMsgInfo {
void* pRetryTmr; // used to dispatch data after a given time duration void* pRetryTmr; // used to dispatch data after a given time duration
TdThreadMutex lock; TdThreadMutex lock;
int8_t inMonitor; int8_t inMonitor;
SArray* pSendInfo; // SArray<SDispatchEntry> SArray* pSendInfo; // SArray<SDispatchEntry>
} SDispatchMsgInfo; } SDispatchMsgInfo;
typedef struct STaskQueue { typedef struct STaskQueue {
@ -345,11 +345,11 @@ typedef struct SSinkRecorder {
} SSinkRecorder; } SSinkRecorder;
typedef struct STaskExecStatisInfo { typedef struct STaskExecStatisInfo {
int64_t created; int64_t created;
int64_t checkTs; int64_t checkTs;
int64_t readyTs; int64_t readyTs;
int64_t startCheckpointId; int64_t startCheckpointId;
int64_t startCheckpointVer; int64_t startCheckpointVer;
int64_t step1Start; int64_t step1Start;
double step1El; double step1El;
@ -432,25 +432,25 @@ struct SStreamTask {
SCheckpointInfo chkInfo; SCheckpointInfo chkInfo;
STaskExec exec; STaskExec exec;
SDataRange dataRange; SDataRange dataRange;
SVersionRange step2Range; // version range used to scan wal, information in dataRange should not modified. SVersionRange step2Range; // version range used to scan wal, information in dataRange should not modified.
SHistoryTaskInfo hTaskInfo; SHistoryTaskInfo hTaskInfo;
STaskId streamTaskId; STaskId streamTaskId;
STaskExecStatisInfo execInfo; STaskExecStatisInfo execInfo;
TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ
SMsgCb* pMsgCb; // msg handle SMsgCb* pMsgCb; // msg handle
SStreamState* pState; // state backend SStreamState* pState; // state backend
SUpstreamInfo upstreamInfo; SUpstreamInfo upstreamInfo;
STaskCheckInfo taskCheckInfo; STaskCheckInfo taskCheckInfo;
// the followings attributes don't be serialized // the followings attributes don't be serialized
SScanhistorySchedInfo schedHistoryInfo; SScanhistorySchedInfo schedHistoryInfo;
int32_t refCnt; int32_t refCnt;
int32_t transferStateAlignCnt; int32_t transferStateAlignCnt;
struct SStreamMeta* pMeta; struct SStreamMeta* pMeta;
SSHashObj* pNameMap; SSHashObj* pNameMap;
void* pBackend; void* pBackend;
int8_t subtableWithoutMd5; int8_t subtableWithoutMd5;
char reserve[256]; char reserve[256];
}; };
typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*); typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*);
@ -459,7 +459,7 @@ typedef struct STaskStartInfo {
int64_t startTs; int64_t startTs;
int64_t readyTs; int64_t readyTs;
int32_t tasksWillRestart; int32_t tasksWillRestart;
int32_t startAllTasks; // restart flag, sentinel to guard the restart procedure. int32_t startAllTasks; // restart flag, sentinel to guard the restart procedure.
SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing
SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed
int64_t elapsedTime; int64_t elapsedTime;
@ -512,7 +512,7 @@ typedef struct SStreamMeta {
SArray* chkpSaved; SArray* chkpSaved;
SArray* chkpInUse; SArray* chkpInUse;
SRWLatch chkpDirLock; SRWLatch chkpDirLock;
void* qHandle; // todo remove it void* qHandle; // todo remove it
void* bkdChkptMgt; void* bkdChkptMgt;
} SStreamMeta; } SStreamMeta;
@ -567,14 +567,14 @@ typedef struct {
} SStreamScanHistoryReq; } SStreamScanHistoryReq;
typedef struct STaskCkptInfo { typedef struct STaskCkptInfo {
int64_t latestId; // saved checkpoint id int64_t latestId; // saved checkpoint id
int64_t latestVer; // saved checkpoint ver int64_t latestVer; // saved checkpoint ver
int64_t latestTime; // latest checkpoint time int64_t latestTime; // latest checkpoint time
int64_t latestSize; // latest checkpoint size int64_t latestSize; // latest checkpoint size
int8_t remoteBackup; // latest checkpoint backup done int8_t remoteBackup; // latest checkpoint backup done
int64_t activeId; // current active checkpoint id int64_t activeId; // current active checkpoint id
int32_t activeTransId; // checkpoint trans id int32_t activeTransId; // checkpoint trans id
int8_t failed; // denote if the checkpoint is failed or not int8_t failed; // denote if the checkpoint is failed or not
} STaskCkptInfo; } STaskCkptInfo;
typedef struct STaskStatusEntry { typedef struct STaskStatusEntry {
@ -589,12 +589,12 @@ typedef struct STaskStatusEntry {
int64_t inputQUnchangeCounter; int64_t inputQUnchangeCounter;
double inputQUsed; // in MiB double inputQUsed; // in MiB
double inputRate; double inputRate;
double procsThroughput; // duration between one element put into input queue and being processed. double procsThroughput; // duration between one element put into input queue and being processed.
double procsTotal; // duration between one element put into input queue and being processed. double procsTotal; // duration between one element put into input queue and being processed.
double outputThroughput; // the size of dispatched result blocks in bytes double outputThroughput; // the size of dispatched result blocks in bytes
double outputTotal; // the size of dispatched result blocks in bytes double outputTotal; // the size of dispatched result blocks in bytes
double sinkQuota; // existed quota size for sink task double sinkQuota; // existed quota size for sink task
double sinkDataSize; // sink to dst data size double sinkDataSize; // sink to dst data size
int64_t startTime; int64_t startTime;
int64_t startCheckpointId; int64_t startCheckpointId;
int64_t startCheckpointVer; int64_t startCheckpointVer;
@ -622,12 +622,12 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId);
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId);
void streamTaskInputFail(SStreamTask* pTask); void streamTaskInputFail(SStreamTask* pTask);
bool streamTaskShouldStop(const SStreamTask* pStatus); bool streamTaskShouldStop(const SStreamTask* pStatus);
bool streamTaskShouldPause(const SStreamTask* pStatus); bool streamTaskShouldPause(const SStreamTask* pStatus);
bool streamTaskIsIdle(const SStreamTask* pTask); bool streamTaskIsIdle(const SStreamTask* pTask);
bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus); bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask); SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask);
@ -636,8 +636,8 @@ void streamTaskResetStatus(SStreamTask* pTask);
void streamTaskSetStatusReady(SStreamTask* pTask); void streamTaskSetStatusReady(SStreamTask* pTask);
ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask); ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask);
bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
// stream task sched // stream task sched
bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask); bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask);
@ -737,10 +737,10 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId,
int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta); int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta);
int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs); void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs);
void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId,
int64_t startTs); int64_t startTs);
void streamMetaClearUpdateTaskList(SStreamMeta* pMeta); void streamMetaClearUpdateTaskList(SStreamMeta* pMeta);
void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId); void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId);
void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRLock(SStreamMeta* pMeta);
void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta);
@ -756,12 +756,13 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
bool streamMetaAllTasksReady(const SStreamMeta* pMeta); bool streamMetaAllTasksReady(const SStreamMeta* pMeta);
// timer // timer
tmr_h streamTimerGetInstance(); tmr_h streamTimerGetInstance();
// checkpoint // checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp); int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp);
int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId, int32_t downstreamTaskId); int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId,
int32_t downstreamTaskId);
int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId); int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId);
int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg); void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg);
@ -786,8 +787,8 @@ int32_t streamTaskRestoreStatus(SStreamTask* pTask);
// stream task retrieve related API // stream task retrieve related API
int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq); int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq);
int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq *req); int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* req);
void streamTaskSendRetrieveRsp(SStreamRetrieveReq *pReq, SRpcMsg* pRsp); void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
source/client/src/clientMain.c
View File

@ -195,6 +195,7 @@ int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
tFreeSGetUserWhiteListRsp(&wlRsp); tFreeSGetUserWhiteListRsp(&wlRsp);
return TSDB_CODE_OUT_OF_MEMORY;
} }
for (int i = 0; i < wlRsp.numWhiteLists; ++i) { for (int i = 0; i < wlRsp.numWhiteLists; ++i) {

204
source/client/src/clientStmt.c
View File

@ -17,11 +17,11 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void**
*pBuf = pTblBuf->pCurBuff; *pBuf = pTblBuf->pCurBuff;
pTblBuf->buffOffset = pTblBuf->buffUnit; pTblBuf->buffOffset = pTblBuf->buffUnit;
} else { } else {
void *buff = taosMemoryMalloc(pTblBuf->buffSize); void* buff = taosMemoryMalloc(pTblBuf->buffSize);
if (NULL == buff) { if (NULL == buff) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(pTblBuf->pBufList, &buff); taosArrayPush(pTblBuf->pBufList, &buff);
pTblBuf->buffIdx++; pTblBuf->buffIdx++;
@ -33,18 +33,18 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void**
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool stmtDequeue(STscStmt* pStmt, SStmtQNode **param) { bool stmtDequeue(STscStmt* pStmt, SStmtQNode** param) {
while (0 == atomic_load_64(&pStmt->queue.qRemainNum)) { while (0 == atomic_load_64(&pStmt->queue.qRemainNum)) {
taosUsleep(1); taosUsleep(1);
return false; return false;
} }
SStmtQNode *orig = pStmt->queue.head; SStmtQNode* orig = pStmt->queue.head;
SStmtQNode *node = pStmt->queue.head->next; SStmtQNode* node = pStmt->queue.head->next;
pStmt->queue.head = pStmt->queue.head->next; pStmt->queue.head = pStmt->queue.head->next;
//taosMemoryFreeClear(orig); // taosMemoryFreeClear(orig);
*param = node; *param = node;
@ -61,7 +61,6 @@ void stmtEnqueue(STscStmt* pStmt, SStmtQNode* param) {
atomic_add_fetch_64(&pStmt->queue.qRemainNum, 1); atomic_add_fetch_64(&pStmt->queue.qRemainNum, 1);
} }
static int32_t stmtCreateRequest(STscStmt* pStmt) { static int32_t stmtCreateRequest(STscStmt* pStmt) {
int32_t code = 0; int32_t code = 0;
@ -93,7 +92,7 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
switch (newStatus) { switch (newStatus) {
case STMT_PREPARE: case STMT_PREPARE:
pStmt->errCode = 0; pStmt->errCode = 0;
break; break;
case STMT_SETTBNAME: case STMT_SETTBNAME:
if (STMT_STATUS_EQ(INIT)) { if (STMT_STATUS_EQ(INIT)) {
@ -265,7 +264,7 @@ int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHa
*pVgHash = pStmt->sql.pVgHash; *pVgHash = pStmt->sql.pVgHash;
pStmt->sql.pVgHash = NULL; pStmt->sql.pVgHash = NULL;
*pBlockHash = pStmt->exec.pBlockHash; *pBlockHash = pStmt->exec.pBlockHash;
pStmt->exec.pBlockHash = NULL; pStmt->exec.pBlockHash = NULL;
@ -325,7 +324,7 @@ int32_t stmtParseSql(STscStmt* pStmt) {
pStmt->stat.parseSqlNum++; pStmt->stat.parseSqlNum++;
STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb)); STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb));
pStmt->sql.siInfo.pQuery = pStmt->sql.pQuery; pStmt->sql.siInfo.pQuery = pStmt->sql.pQuery;
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
if (pStmt->sql.pQuery->pRoot && 0 == pStmt->sql.type) { if (pStmt->sql.pQuery->pRoot && 0 == pStmt->sql.type) {
@ -338,15 +337,16 @@ int32_t stmtParseSql(STscStmt* pStmt) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
STableDataCxt** pSrc = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); STableDataCxt** pSrc =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pSrc || NULL == *pSrc) { if (NULL == pSrc || NULL == *pSrc) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
STableDataCxt* pTableCtx = *pSrc; STableDataCxt* pTableCtx = *pSrc;
if (pStmt->sql.stbInterlaceMode) { if (pStmt->sql.stbInterlaceMode) {
int16_t lastIdx = -1; int16_t lastIdx = -1;
for (int32_t i = 0; i < pTableCtx->boundColsInfo.numOfBound; ++i) { for (int32_t i = 0; i < pTableCtx->boundColsInfo.numOfBound; ++i) {
if (pTableCtx->boundColsInfo.pColIndex[i] < lastIdx) { if (pTableCtx->boundColsInfo.pColIndex[i] < lastIdx) {
pStmt->sql.stbInterlaceMode = false; pStmt->sql.stbInterlaceMode = false;
@ -381,7 +381,8 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) {
qDestroyBoundColInfo(pStmt->bInfo.boundTags); qDestroyBoundColInfo(pStmt->bInfo.boundTags);
taosMemoryFreeClear(pStmt->bInfo.boundTags); taosMemoryFreeClear(pStmt->bInfo.boundTags);
} }
pStmt->bInfo.stbFName[0] = 0;; pStmt->bInfo.stbFName[0] = 0;
;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -457,12 +458,12 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void stmtFreeTbBuf(void *buf) { void stmtFreeTbBuf(void* buf) {
void* pBuf = *(void**)buf; void* pBuf = *(void**)buf;
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
} }
void stmtFreeTbCols(void *buf) { void stmtFreeTbCols(void* buf) {
SArray* pCols = *(SArray**)buf; SArray* pCols = *(SArray**)buf;
taosArrayDestroy(pCols); taosArrayDestroy(pCols);
} }
@ -505,19 +506,17 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
memset(&pStmt->sql, 0, sizeof(pStmt->sql)); memset(&pStmt->sql, 0, sizeof(pStmt->sql));
pStmt->sql.siInfo.tableColsReady = true; pStmt->sql.siInfo.tableColsReady = true;
STMT_DLOG_E("end to free SQL info"); STMT_DLOG_E("end to free SQL info");
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t stmtTryAddTableVgroupInfo(STscStmt* pStmt, int32_t* vgId) { int32_t stmtTryAddTableVgroupInfo(STscStmt* pStmt, int32_t* vgId) {
if (*vgId >= 0 && taosHashGet(pStmt->sql.pVgHash, (const char*)vgId, sizeof(*vgId))) { if (*vgId >= 0 && taosHashGet(pStmt->sql.pVgHash, (const char*)vgId, sizeof(*vgId))) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter,
.requestId = pStmt->exec.pRequest->requestId, .requestId = pStmt->exec.pRequest->requestId,
@ -528,17 +527,17 @@ int32_t stmtTryAddTableVgroupInfo(STscStmt* pStmt, int32_t* vgId) {
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
code = taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo)); code =
taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo));
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
*vgId = vgInfo.vgId; *vgId = vgInfo.vgId;
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_SUCCESS;
}
int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid, int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid,
uint64_t suid, int32_t vgId) { uint64_t suid, int32_t vgId) {
@ -554,7 +553,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
if (pStmt->sql.stbInterlaceMode && pStmt->sql.siInfo.pDataCtx) { if (pStmt->sql.stbInterlaceMode && pStmt->sql.siInfo.pDataCtx) {
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
pStmt->bInfo.inExecCache = false; pStmt->bInfo.inExecCache = false;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
pStmt->bInfo.needParse = true; pStmt->bInfo.needParse = true;
@ -589,7 +588,6 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (pStmt->sql.autoCreateTbl) { if (pStmt->sql.autoCreateTbl) {
SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid));
if (pCache) { if (pCache) {
@ -615,8 +613,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
} }
uint64_t uid, suid; uint64_t uid, suid;
int32_t vgId; int32_t vgId;
int8_t tableType; int8_t tableType;
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter,
@ -638,12 +636,12 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
uid = pTableMeta->uid; uid = pTableMeta->uid;
suid = pTableMeta->suid; suid = pTableMeta->suid;
tableType = pTableMeta->tableType; tableType = pTableMeta->tableType;
pStmt->bInfo.tbVgId = pTableMeta->vgId; pStmt->bInfo.tbVgId = pTableMeta->vgId;
vgId = pTableMeta->vgId; vgId = pTableMeta->vgId;
taosMemoryFree(pTableMeta); taosMemoryFree(pTableMeta);
uint64_t cacheUid = (TSDB_CHILD_TABLE == tableType) ? suid : uid; uint64_t cacheUid = (TSDB_CHILD_TABLE == tableType) ? suid : uid;
if (uid == pStmt->bInfo.tbUid) { if (uid == pStmt->bInfo.tbUid) {
@ -675,7 +673,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid)); SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid));
if (pCache) { if (pCache) {
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
@ -720,7 +718,6 @@ int32_t stmtResetStmt(STscStmt* pStmt) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) { int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) {
SStmtQNode* pParam = (SStmtQNode*)param; SStmtQNode* pParam = (SStmtQNode*)param;
@ -729,20 +726,20 @@ int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) {
SArray** p = (SArray**)TARRAY_GET_ELEM(pStmt->sql.siInfo.pTableCols, i); SArray** p = (SArray**)TARRAY_GET_ELEM(pStmt->sql.siInfo.pTableCols, i);
*p = taosArrayInit(20, POINTER_BYTES); *p = taosArrayInit(20, POINTER_BYTES);
} }
atomic_store_8((int8_t*)&pStmt->sql.siInfo.tableColsReady, true); atomic_store_8((int8_t*)&pStmt->sql.siInfo.tableColsReady, true);
} else { } else {
STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock, &pStmt->sql.siInfo)); STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock,
&pStmt->sql.siInfo));
//taosMemoryFree(pParam->pTbData); // taosMemoryFree(pParam->pTbData);
atomic_sub_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1); atomic_sub_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void* stmtBindThreadFunc(void* param) {
void *stmtBindThreadFunc(void *param) {
setThreadName("stmtBind"); setThreadName("stmtBind");
qInfo("stmt bind thread started"); qInfo("stmt bind thread started");
@ -750,15 +747,15 @@ void *stmtBindThreadFunc(void *param) {
STscStmt* pStmt = (STscStmt*)param; STscStmt* pStmt = (STscStmt*)param;
while (true) { while (true) {
if (atomic_load_8((int8_t *)&pStmt->queue.stopQueue)) { if (atomic_load_8((int8_t*)&pStmt->queue.stopQueue)) {
break; break;
} }
SStmtQNode *asyncParam = NULL; SStmtQNode* asyncParam = NULL;
if (!stmtDequeue(pStmt, &asyncParam)) { if (!stmtDequeue(pStmt, &asyncParam)) {
continue; continue;
} }
stmtAsyncOutput(pStmt, asyncParam); stmtAsyncOutput(pStmt, asyncParam);
} }
@ -767,7 +764,6 @@ void *stmtBindThreadFunc(void *param) {
return NULL; return NULL;
} }
int32_t stmtStartBindThread(STscStmt* pStmt) { int32_t stmtStartBindThread(STscStmt* pStmt) {
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
@ -798,7 +794,7 @@ int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) {
if (NULL == pTblBuf->pBufList) { if (NULL == pTblBuf->pBufList) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
void *buff = taosMemoryMalloc(pTblBuf->buffSize); void* buff = taosMemoryMalloc(pTblBuf->buffSize);
if (NULL == buff) { if (NULL == buff) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -815,7 +811,7 @@ int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) {
TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid, TAOS_STMT_OPTIONS* pOptions) { TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid, TAOS_STMT_OPTIONS* pOptions) {
STscObj* pObj = (STscObj*)taos; STscObj* pObj = (STscObj*)taos;
STscStmt* pStmt = NULL; STscStmt* pStmt = NULL;
int32_t code = 0; int32_t code = 0;
pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); pStmt = taosMemoryCalloc(1, sizeof(STscStmt));
if (NULL == pStmt) { if (NULL == pStmt) {
@ -915,22 +911,21 @@ int32_t stmtInitStbInterlaceTableInfo(STscStmt* pStmt) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
STableDataCxt* pDst = NULL; STableDataCxt* pDst = NULL;
STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc, true)); STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc, true));
pStmt->sql.siInfo.pDataCtx = pDst; pStmt->sql.siInfo.pDataCtx = pDst;
SArray* pTblCols = NULL; SArray* pTblCols = NULL;
for (int32_t i = 0; i < STMT_TABLE_COLS_NUM; i++) { for (int32_t i = 0; i < STMT_TABLE_COLS_NUM; i++) {
pTblCols = taosArrayInit(20, POINTER_BYTES); pTblCols = taosArrayInit(20, POINTER_BYTES);
if (NULL == pTblCols) { if (NULL == pTblCols) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols); taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols);
} }
pStmt->sql.siInfo.boundTags = pStmt->bInfo.boundTags;
pStmt->sql.siInfo.boundTags = pStmt->bInfo.boundTags;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -973,13 +968,13 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb,
pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName); tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName);
STMT_ERR_RET(stmtGetFromCache(pStmt)); STMT_ERR_RET(stmtGetFromCache(pStmt));
if (pStmt->bInfo.needParse) { if (pStmt->bInfo.needParse) {
strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1);
pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0;
STMT_ERR_RET(stmtParseSql(pStmt)); STMT_ERR_RET(stmtParseSql(pStmt));
} }
} else { } else {
@ -1054,7 +1049,8 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields
if (pStmt->sql.stbInterlaceMode) { if (pStmt->sql.stbInterlaceMode) {
pDataBlock = &pStmt->sql.siInfo.pDataCtx; pDataBlock = &pStmt->sql.siInfo.pDataCtx;
} else { } else {
pDataBlock = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); pDataBlock =
(STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName));
if (NULL == pDataBlock) { if (NULL == pDataBlock) {
tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName);
STMT_ERR_RET(TSDB_CODE_APP_ERROR); STMT_ERR_RET(TSDB_CODE_APP_ERROR);
@ -1072,7 +1068,7 @@ SArray* stmtGetFreeCol(STscStmt* pStmt, int32_t* idx) {
if (pStmt->exec.smInfo.pColIdx >= STMT_COL_BUF_SIZE) { if (pStmt->exec.smInfo.pColIdx >= STMT_COL_BUF_SIZE) {
pStmt->exec.smInfo.pColIdx = 0; pStmt->exec.smInfo.pColIdx = 0;
} }
if ((pStmt->exec.smInfo.pColIdx + 1) == atomic_load_32(&pStmt->exec.smInfo.pColFreeIdx)) { if ((pStmt->exec.smInfo.pColIdx + 1) == atomic_load_32(&pStmt->exec.smInfo.pColFreeIdx)) {
taosUsleep(1); taosUsleep(1);
continue; continue;
@ -1086,38 +1082,40 @@ SArray* stmtGetFreeCol(STscStmt* pStmt, int32_t* idx) {
int32_t stmtAppendTablePostHandle(STscStmt* pStmt, SStmtQNode* param) { int32_t stmtAppendTablePostHandle(STscStmt* pStmt, SStmtQNode* param) {
if (NULL == pStmt->sql.siInfo.pVgroupHash) { if (NULL == pStmt->sql.siInfo.pVgroupHash) {
pStmt->sql.siInfo.pVgroupHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); pStmt->sql.siInfo.pVgroupHash =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
} }
if (NULL == pStmt->sql.siInfo.pVgroupList) { if (NULL == pStmt->sql.siInfo.pVgroupList) {
pStmt->sql.siInfo.pVgroupList = taosArrayInit(64, POINTER_BYTES); pStmt->sql.siInfo.pVgroupList = taosArrayInit(64, POINTER_BYTES);
} }
if (NULL == pStmt->sql.siInfo.pRequest) { if (NULL == pStmt->sql.siInfo.pRequest) {
STMT_ERR_RET(buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false, (SRequestObj**)&pStmt->sql.siInfo.pRequest, STMT_ERR_RET(buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false,
pStmt->reqid)); (SRequestObj**)&pStmt->sql.siInfo.pRequest, pStmt->reqid));
if (pStmt->reqid != 0) { if (pStmt->reqid != 0) {
pStmt->reqid++; pStmt->reqid++;
} }
pStmt->exec.pRequest->syncQuery = true; pStmt->exec.pRequest->syncQuery = true;
pStmt->sql.siInfo.requestId = ((SRequestObj*)pStmt->sql.siInfo.pRequest)->requestId; pStmt->sql.siInfo.requestId = ((SRequestObj*)pStmt->sql.siInfo.pRequest)->requestId;
pStmt->sql.siInfo.requestSelf = ((SRequestObj*)pStmt->sql.siInfo.pRequest)->self; pStmt->sql.siInfo.requestSelf = ((SRequestObj*)pStmt->sql.siInfo.pRequest)->self;
} }
if (!pStmt->sql.siInfo.tbFromHash && pStmt->sql.siInfo.firstName[0] && 0 == strcmp(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName)) { if (!pStmt->sql.siInfo.tbFromHash && pStmt->sql.siInfo.firstName[0] &&
0 == strcmp(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName)) {
pStmt->sql.siInfo.tbFromHash = true; pStmt->sql.siInfo.tbFromHash = true;
} }
if (0 == pStmt->sql.siInfo.firstName[0]) { if (0 == pStmt->sql.siInfo.firstName[0]) {
strcpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName); strcpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName);
} }
param->tblData.getFromHash = pStmt->sql.siInfo.tbFromHash; param->tblData.getFromHash = pStmt->sql.siInfo.tbFromHash;
param->next = NULL; param->next = NULL;
atomic_add_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1); atomic_add_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1);
stmtEnqueue(pStmt, param); stmtEnqueue(pStmt, param);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1135,7 +1133,7 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt* pStmt, SArray**
if (NULL == pTblCols) { if (NULL == pTblCols) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols); taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols);
} }
} }
@ -1146,10 +1144,10 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt* pStmt, SArray**
int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
int32_t code = 0; int32_t code = 0;
int64_t startUs = taosGetTimestampUs(); int64_t startUs = taosGetTimestampUs();
STMT_DLOG("start to bind stmt data, colIdx: %d", colIdx); STMT_DLOG("start to bind stmt data, colIdx: %d", colIdx);
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
@ -1232,7 +1230,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
} }
int64_t startUs2 = taosGetTimestampUs(); int64_t startUs2 = taosGetTimestampUs();
pStmt->stat.bindDataUs1 += startUs2 - startUs; pStmt->stat.bindDataUs1 += startUs2 - startUs;
SStmtQNode* param = NULL; SStmtQNode* param = NULL;
if (pStmt->sql.stbInterlaceMode) { if (pStmt->sql.stbInterlaceMode) {
@ -1240,25 +1238,27 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
STMT_ERR_RET(stmtGetTableColsFromCache(pStmt, &param->tblData.aCol)); STMT_ERR_RET(stmtGetTableColsFromCache(pStmt, &param->tblData.aCol));
taosArrayClear(param->tblData.aCol); taosArrayClear(param->tblData.aCol);
//param->tblData.aCol = taosArrayInit(20, POINTER_BYTES); // param->tblData.aCol = taosArrayInit(20, POINTER_BYTES);
param->restoreTbCols = false; param->restoreTbCols = false;
strcpy(param->tblData.tbName, pStmt->bInfo.tbName); strcpy(param->tblData.tbName, pStmt->bInfo.tbName);
} }
int64_t startUs3 = taosGetTimestampUs(); int64_t startUs3 = taosGetTimestampUs();
pStmt->stat.bindDataUs2 += startUs3 - startUs2; pStmt->stat.bindDataUs2 += startUs3 - startUs2;
SArray* pCols = pStmt->sql.stbInterlaceMode ? param->tblData.aCol : (*pDataBlock)->pData->aCol; SArray* pCols = pStmt->sql.stbInterlaceMode ? param->tblData.aCol : (*pDataBlock)->pData->aCol;
if (colIdx < 0) { if (colIdx < 0) {
if (pStmt->sql.stbInterlaceMode) { if (pStmt->sql.stbInterlaceMode) {
(*pDataBlock)->pData->flags = 0; (*pDataBlock)->pData->flags = 0;
code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo); code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo);
} else { } else {
code = qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); code =
qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen);
} }
if (code) { if (code) {
tscError("qBindStmtColsValue failed, error:%s", tstrerror(code)); tscError("qBindStmtColsValue failed, error:%s", tstrerror(code));
STMT_ERR_RET(code); STMT_ERR_RET(code);
@ -1268,7 +1268,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
tscError("bind single column not allowed in stb insert mode"); tscError("bind single column not allowed in stb insert mode");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
} }
if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) { if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) {
tscError("bind column index not in sequence"); tscError("bind column index not in sequence");
STMT_ERR_RET(TSDB_CODE_APP_ERROR); STMT_ERR_RET(TSDB_CODE_APP_ERROR);
@ -1289,13 +1289,13 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
} }
int64_t startUs4 = taosGetTimestampUs(); int64_t startUs4 = taosGetTimestampUs();
pStmt->stat.bindDataUs3 += startUs4 - startUs3; pStmt->stat.bindDataUs3 += startUs4 - startUs3;
if (pStmt->sql.stbInterlaceMode) { if (pStmt->sql.stbInterlaceMode) {
STMT_ERR_RET(stmtAppendTablePostHandle(pStmt, param)); STMT_ERR_RET(stmtAppendTablePostHandle(pStmt, param));
} }
pStmt->stat.bindDataUs4 += taosGetTimestampUs() - startUs4; pStmt->stat.bindDataUs4 += taosGetTimestampUs() - startUs4;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1314,12 +1314,12 @@ int stmtAddBatch(TAOS_STMT* stmt) {
pStmt->stat.addBatchUs += startUs2 - startUs; pStmt->stat.addBatchUs += startUs2 - startUs;
pStmt->sql.siInfo.tableColsReady = false; pStmt->sql.siInfo.tableColsReady = false;
SStmtQNode* param = NULL; SStmtQNode* param = NULL;
STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)&param)); STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)&param));
param->restoreTbCols = true; param->restoreTbCols = true;
param->next = NULL; param->next = NULL;
stmtEnqueue(pStmt, param); stmtEnqueue(pStmt, param);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1394,7 +1394,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) {
int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta); int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta);
pStmt->stat.ctgGetTbMetaNum++; pStmt->stat.ctgGetTbMetaNum++;
taos_free_result(pStmt->exec.pRequest); taos_free_result(pStmt->exec.pRequest);
pStmt->exec.pRequest = NULL; pStmt->exec.pRequest = NULL;
@ -1424,13 +1424,14 @@ int stmtStaticModeExec(TAOS_STMT* stmt) {
if (pStmt->sql.staticMode) { if (pStmt->sql.staticMode) {
return TSDB_CODE_TSC_STMT_API_ERROR; return TSDB_CODE_TSC_STMT_API_ERROR;
} }
STMT_DLOG_E("start to exec"); STMT_DLOG_E("start to exec");
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
STMT_ERR_RET(qBuildStmtOutputFromTbList(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pTbBlkList, pStmt->exec.pCurrBlock, pStmt->exec.tbBlkNum)); STMT_ERR_RET(qBuildStmtOutputFromTbList(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pTbBlkList,
pStmt->exec.pCurrBlock, pStmt->exec.tbBlkNum));
launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL);
if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) {
@ -1452,7 +1453,7 @@ int stmtStaticModeExec(TAOS_STMT* stmt) {
_return: _return:
stmtCleanExecInfo(pStmt, (code ? false : true), false); stmtCleanExecInfo(pStmt, (code ? false : true), false);
tFreeSSubmitRsp(pRsp); tFreeSSubmitRsp(pRsp);
++pStmt->sql.runTimes; ++pStmt->sql.runTimes;
@ -1481,7 +1482,7 @@ int stmtExec(TAOS_STMT* stmt) {
taosUsleep(1); taosUsleep(1);
} }
pStmt->stat.execWaitUs += taosGetTimestampUs() - startTs; pStmt->stat.execWaitUs += taosGetTimestampUs() - startTs;
STMT_ERR_RET(qBuildStmtFinOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->sql.siInfo.pVgroupList)); STMT_ERR_RET(qBuildStmtFinOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->sql.siInfo.pVgroupList));
taosHashCleanup(pStmt->sql.siInfo.pVgroupHash); taosHashCleanup(pStmt->sql.siInfo.pVgroupHash);
pStmt->sql.siInfo.pVgroupHash = NULL; pStmt->sql.siInfo.pVgroupHash = NULL;
@ -1494,7 +1495,7 @@ int stmtExec(TAOS_STMT* stmt) {
STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash)); STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash));
} }
launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL);
} }
@ -1521,7 +1522,7 @@ _return:
} }
stmtCleanExecInfo(pStmt, (code ? false : true), false); stmtCleanExecInfo(pStmt, (code ? false : true), false);
tFreeSSubmitRsp(pRsp); tFreeSSubmitRsp(pRsp);
++pStmt->sql.runTimes; ++pStmt->sql.runTimes;
@ -1544,13 +1545,16 @@ int stmtClose(TAOS_STMT* stmt) {
pStmt->bindThreadInUse = false; pStmt->bindThreadInUse = false;
} }
STMT_DLOG("stmt %p closed, stbInterlaceMode: %d, statInfo: ctgGetTbMetaNum=>%" PRId64 ", getCacheTbInfo=>%" PRId64 ", parseSqlNum=>%" PRId64 STMT_DLOG("stmt %p closed, stbInterlaceMode: %d, statInfo: ctgGetTbMetaNum=>%" PRId64 ", getCacheTbInfo=>%" PRId64
", pStmt->stat.bindDataNum=>%" PRId64 ", settbnameAPI:%u, bindAPI:%u, addbatchAPI:%u, execAPI:%u" ", parseSqlNum=>%" PRId64 ", pStmt->stat.bindDataNum=>%" PRId64
", setTbNameUs:%" PRId64 ", bindDataUs:%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 " addBatchUs:%" PRId64 ", execWaitUs:%" PRId64 ", execUseUs:%" PRId64, ", settbnameAPI:%u, bindAPI:%u, addbatchAPI:%u, execAPI:%u"
pStmt, pStmt->sql.stbInterlaceMode, pStmt->stat.ctgGetTbMetaNum, pStmt->stat.getCacheTbInfo, pStmt->stat.parseSqlNum, pStmt->stat.bindDataNum, ", setTbNameUs:%" PRId64 ", bindDataUs:%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 " addBatchUs:%" PRId64
pStmt->seqIds[STMT_SETTBNAME], pStmt->seqIds[STMT_BIND], pStmt->seqIds[STMT_ADD_BATCH], pStmt->seqIds[STMT_EXECUTE], ", execWaitUs:%" PRId64 ", execUseUs:%" PRId64,
pStmt->stat.setTbNameUs, pStmt->stat.bindDataUs1, pStmt->stat.bindDataUs2, pStmt->stat.bindDataUs3, pStmt->stat.bindDataUs4, pStmt, pStmt->sql.stbInterlaceMode, pStmt->stat.ctgGetTbMetaNum, pStmt->stat.getCacheTbInfo,
pStmt->stat.addBatchUs, pStmt->stat.execWaitUs, pStmt->stat.execUseUs); pStmt->stat.parseSqlNum, pStmt->stat.bindDataNum, pStmt->seqIds[STMT_SETTBNAME], pStmt->seqIds[STMT_BIND],
pStmt->seqIds[STMT_ADD_BATCH], pStmt->seqIds[STMT_EXECUTE], pStmt->stat.setTbNameUs,
pStmt->stat.bindDataUs1, pStmt->stat.bindDataUs2, pStmt->stat.bindDataUs3, pStmt->stat.bindDataUs4,
pStmt->stat.addBatchUs, pStmt->stat.execWaitUs, pStmt->stat.execUseUs);
stmtCleanSQLInfo(pStmt); stmtCleanSQLInfo(pStmt);
taosMemoryFree(stmt); taosMemoryFree(stmt);
@ -1589,9 +1593,9 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
} }
int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = 0; int32_t code = 0;
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
int32_t preCode = pStmt->errCode; int32_t preCode = pStmt->errCode;
STMT_DLOG_E("start to get tag fields"); STMT_DLOG_E("start to get tag fields");
@ -1622,14 +1626,14 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
_return: _return:
pStmt->errCode = preCode; pStmt->errCode = preCode;
return code; return code;
} }
int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
int32_t code = 0; int32_t code = 0;
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
int32_t preCode = pStmt->errCode; int32_t preCode = pStmt->errCode;
STMT_DLOG_E("start to get col fields"); STMT_DLOG_E("start to get col fields");
@ -1683,7 +1687,7 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
} }
STMT_ERR_RET(stmtCreateRequest(pStmt)); STMT_ERR_RET(stmtCreateRequest(pStmt));
if (pStmt->bInfo.needParse) { if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt)); STMT_ERR_RET(stmtParseSql(pStmt));
} }

116
source/client/src/clientTmq.c
View File

@ -107,7 +107,7 @@ struct tmq_t {
STaosQueue* mqueue; // queue of rsp STaosQueue* mqueue; // queue of rsp
STaosQall* qall; STaosQall* qall;
STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit
tsem2_t rspSem; tsem2_t rspSem;
}; };
typedef struct SAskEpInfo { typedef struct SAskEpInfo {
@ -681,7 +681,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName,
numOfVgroups); numOfVgroups);
for (int32_t j = 0; j < numOfVgroups; j++) { for (int32_t j = 0; j < numOfVgroups; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
@ -703,19 +703,19 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
} }
tscDebug("consumer:0x%" PRIx64 tscDebug("consumer:0x%" PRIx64
" topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d", " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d",
tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups); tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups);
tOffsetCopy(&pVg->offsetInfo.committedOffset, &pVg->offsetInfo.endOffset); tOffsetCopy(&pVg->offsetInfo.committedOffset, &pVg->offsetInfo.endOffset);
} else { } else {
tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d",
tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups);
} }
} }
} }
taosRUnLockLatch(&tmq->lock); taosRUnLockLatch(&tmq->lock);
tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1,
numOfTopics); numOfTopics);
// request is sent // request is sent
if (pParamSet->waitingRspNum != 1) { if (pParamSet->waitingRspNum != 1) {
@ -830,7 +830,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
char buf[TSDB_OFFSET_LEN] = {0}; char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset);
tscDebug("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64, tscDebug("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64,
tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows); tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows);
} }
} }
taosRUnLockLatch(&tmq->lock); taosRUnLockLatch(&tmq->lock);
@ -955,7 +955,7 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
taosMemoryFreeClear(pRsp->pEpset); taosMemoryFreeClear(pRsp->pEpset);
tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp);
} }
} }
void tmqClearUnhandleMsg(tmq_t* tmq) { void tmqClearUnhandleMsg(tmq_t* tmq) {
@ -1021,17 +1021,16 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
taosSsleep(2); // sleep 2s for hb to send offset and rows to server taosSsleep(2); // sleep 2s for hb to send offset and rows to server
tmq_list_t* lst = tmq_list_new(); tmq_list_t* lst = tmq_list_new();
int32_t rsp = tmq_subscribe(tmq, lst); int32_t rsp = tmq_subscribe(tmq, lst);
tmq_list_destroy(lst); tmq_list_destroy(lst);
return rsp; return rsp;
} }
static void freeClientVg(void* param){ static void freeClientVg(void* param) {
SMqClientVg* pVg = param; SMqClientVg* pVg = param;
tOffsetDestroy(&pVg->offsetInfo.endOffset); tOffsetDestroy(&pVg->offsetInfo.endOffset);
tOffsetDestroy(&pVg->offsetInfo.beginOffset); tOffsetDestroy(&pVg->offsetInfo.beginOffset);
tOffsetDestroy(&pVg->offsetInfo.committedOffset); tOffsetDestroy(&pVg->offsetInfo.committedOffset);
} }
static void freeClientVgImpl(void* param) { static void freeClientVgImpl(void* param) {
SMqClientTopic* pTopic = param; SMqClientTopic* pTopic = param;
@ -1282,8 +1281,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
int32_t retryCnt = 0; int32_t retryCnt = 0;
while ((code = syncAskEp(tmq)) != 0) { while ((code = syncAskEp(tmq)) != 0) {
if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", tmq->consumerId, strerror(code)); tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s",
if(code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { tmq->consumerId, strerror(code));
if (code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
code = 0; code = 0;
} }
goto FAIL; goto FAIL;
@ -1379,7 +1379,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
goto END; goto END;
} }
if(pMsg->pData == NULL){ if (pMsg->pData == NULL) {
tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId); tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId);
code = TSDB_CODE_TSC_INTERNAL_ERROR; code = TSDB_CODE_TSC_INTERNAL_ERROR;
goto END; goto END;
@ -1407,7 +1407,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0){ if (tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId); taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -1423,7 +1423,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
} else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp) < 0){ if (tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp) < 0) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId); taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -1434,7 +1434,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
} else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0){ if (tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId); taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -1445,7 +1445,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
} else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0){ if (tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId); taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -1529,11 +1529,11 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
clientVg.offsetInfo.walVerBegin = -1; clientVg.offsetInfo.walVerBegin = -1;
clientVg.offsetInfo.walVerEnd = -1; clientVg.offsetInfo.walVerEnd = -1;
clientVg.seekUpdated = false; clientVg.seekUpdated = false;
if(pInfo) { if (pInfo) {
tOffsetCopy(&clientVg.offsetInfo.endOffset, &pInfo->currentOffset); tOffsetCopy(&clientVg.offsetInfo.endOffset, &pInfo->currentOffset);
tOffsetCopy(&clientVg.offsetInfo.committedOffset, &pInfo->commitOffset); tOffsetCopy(&clientVg.offsetInfo.committedOffset, &pInfo->commitOffset);
tOffsetCopy(&clientVg.offsetInfo.beginOffset, &pInfo->seekOffset); tOffsetCopy(&clientVg.offsetInfo.beginOffset, &pInfo->seekOffset);
}else{ } else {
clientVg.offsetInfo.endOffset = offsetNew; clientVg.offsetInfo.endOffset = offsetNew;
clientVg.offsetInfo.committedOffset = offsetNew; clientVg.offsetInfo.committedOffset = offsetNew;
clientVg.offsetInfo.beginOffset = offsetNew; clientVg.offsetInfo.beginOffset = offsetNew;
@ -1548,7 +1548,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
int32_t topicNumGet = taosArrayGetSize(pRsp->topics); int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) { if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) {
tscDebug("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tmq->consumerId, tscDebug("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tmq->consumerId,
tmq->epoch, epoch, topicNumGet); tmq->epoch, epoch, topicNumGet);
if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
} }
@ -1642,7 +1642,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl
SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj));
if(pRspObj == NULL) { if (pRspObj == NULL) {
return NULL; return NULL;
} }
pRspObj->resType = RES_TYPE__TMQ_META; pRspObj->resType = RES_TYPE__TMQ_META;
@ -1656,7 +1656,7 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj)); SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj));
if(pRspObj == NULL) { if (pRspObj == NULL) {
return NULL; return NULL;
} }
pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META; pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META;
@ -1669,10 +1669,11 @@ SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper)
return pRspObj; return pRspObj;
} }
void changeByteEndian(char* pData){ void changeByteEndian(char* pData) {
char* p = pData; char* p = pData;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | version: // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column
// length | version:
int32_t blockVersion = *(int32_t*)p; int32_t blockVersion = *(int32_t*)p;
ASSERT(blockVersion == BLOCK_VERSION_1); ASSERT(blockVersion == BLOCK_VERSION_1);
*(int32_t*)p = BLOCK_VERSION_2; *(int32_t*)p = BLOCK_VERSION_2;
@ -1693,23 +1694,24 @@ void changeByteEndian(char* pData){
} }
} }
static void tmqGetRawDataRowsPrecisionFromRes(void *pRetrieve, void** rawData, int64_t *rows, int32_t *precision){ static void tmqGetRawDataRowsPrecisionFromRes(void* pRetrieve, void** rawData, int64_t* rows, int32_t* precision) {
if(*(int64_t*)pRetrieve == 0){ if (*(int64_t*)pRetrieve == 0) {
*rawData = ((SRetrieveTableRsp*)pRetrieve)->data; *rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
*rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows); *rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows);
if(precision != NULL){ if (precision != NULL) {
*precision = ((SRetrieveTableRsp*)pRetrieve)->precision; *precision = ((SRetrieveTableRsp*)pRetrieve)->precision;
} }
}else if(*(int64_t*)pRetrieve == 1){ } else if (*(int64_t*)pRetrieve == 1) {
*rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data; *rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
*rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows); *rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows);
if(precision != NULL){ if (precision != NULL) {
*precision = ((SRetrieveTableRspForTmq*)pRetrieve)->precision; *precision = ((SRetrieveTableRspForTmq*)pRetrieve)->precision;
} }
} }
} }
static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) { static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows,
SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) {
(*numOfRows) = 0; (*numOfRows) = 0;
tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
@ -1727,8 +1729,8 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
} }
// extract the rows in this data packet // extract the rows in this data packet
for (int32_t i = 0; i < pDataRsp->blockNum; ++i) { for (int32_t i = 0; i < pDataRsp->blockNum; ++i) {
void* pRetrieve = taosArrayGetP(pDataRsp->blockData, i); void* pRetrieve = taosArrayGetP(pDataRsp->blockData, i);
void* rawData = NULL; void* rawData = NULL;
int64_t rows = 0; int64_t rows = 0;
// deal with compatibility // deal with compatibility
tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, NULL); tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, NULL);
@ -1736,9 +1738,9 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
pVg->numOfRows += rows; pVg->numOfRows += rows;
(*numOfRows) += rows; (*numOfRows) += rows;
changeByteEndian(rawData); changeByteEndian(rawData);
if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
if(schema){ if (schema) {
taosArrayPush(pDataRsp->blockSchema, &schema); taosArrayPush(pDataRsp->blockSchema, &schema);
} }
} }
@ -1747,7 +1749,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
if(pRspObj == NULL){ if (pRspObj == NULL) {
return NULL; return NULL;
} }
pRspObj->common.resType = RES_TYPE__TMQ; pRspObj->common.resType = RES_TYPE__TMQ;
@ -1758,7 +1760,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg,
SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
if(pRspObj == NULL){ if (pRspObj == NULL) {
return NULL; return NULL;
} }
pRspObj->common.resType = RES_TYPE__TMQ_METADATA; pRspObj->common.resType = RES_TYPE__TMQ_METADATA;
@ -1790,6 +1792,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) {
code = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
taosMemoryFreeClear(msg);
goto FAIL; goto FAIL;
} }
@ -1858,7 +1861,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
} }
for (int j = 0; j < numOfVg; j++) { for (int j = 0; j < numOfVg; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs;
if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms
tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId,
tmq->epoch, pVg->vgId); tmq->epoch, pVg->vgId);
@ -1956,7 +1959,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch); int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->dataRsp; SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->dataRsp;
if (pDataRsp->head.epoch == consumerEpoch) { if (pDataRsp->head.epoch == consumerEpoch) {
@ -2096,7 +2099,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch); int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp; SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp;
if (pDataRsp->head.epoch == consumerEpoch) { if (pDataRsp->head.epoch == consumerEpoch) {
taosWLockLatch(&tmq->lock); taosWLockLatch(&tmq->lock);
@ -2112,9 +2115,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
return NULL; return NULL;
} }
updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever,
pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId, tmq->consumerId, pDataRsp->blockNum != 0);
pDataRsp->blockNum != 0);
if (pDataRsp->blockNum == 0) { if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64,
@ -2130,15 +2132,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
// build rsp // build rsp
int64_t numOfRows = 0; int64_t numOfRows = 0;
void* pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows); void* pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
tmq->totalRows += numOfRows; tmq->totalRows += numOfRows;
char buf[TSDB_OFFSET_LEN] = {0}; char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
tmq->totalRows, pollRspWrapper->reqId); pollRspWrapper->reqId);
taosFreeQitem(pRspWrapper); taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock); taosWUnLockLatch(&tmq->lock);
@ -2266,7 +2268,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
taosSsleep(2); // sleep 2s for hb to send offset and rows to server taosSsleep(2); // sleep 2s for hb to send offset and rows to server
tmq_list_t* lst = tmq_list_new(); tmq_list_t* lst = tmq_list_new();
int32_t code = tmq_subscribe(tmq, lst); int32_t code = tmq_subscribe(tmq, lst);
tmq_list_destroy(lst); tmq_list_destroy(lst);
if (code != 0) { if (code != 0) {
return code; return code;
@ -2285,9 +2287,9 @@ const char* tmq_err2str(int32_t err) {
} else if (err == -1) { } else if (err == -1) {
return "fail"; return "fail";
} else { } else {
if(*(taosGetErrMsg()) == 0){ if (*(taosGetErrMsg()) == 0) {
return tstrerror(err); return tstrerror(err);
} else{ } else {
snprintf(taosGetErrMsgReturn(), ERR_MSG_LEN, "%s,detail:%s", tstrerror(err), taosGetErrMsg()); snprintf(taosGetErrMsgReturn(), ERR_MSG_LEN, "%s,detail:%s", tstrerror(err), taosGetErrMsg());
return (const char*)taosGetErrMsgReturn(); return (const char*)taosGetErrMsgReturn();
} }
@ -2359,8 +2361,8 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) {
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
STqOffsetVal* pOffset = &common->reqOffset; STqOffsetVal* pOffset = &common->reqOffset;
if (common->reqOffset.type == TMQ_OFFSET__LOG) { if (common->reqOffset.type == TMQ_OFFSET__LOG) {
return common->reqOffset.version; return common->reqOffset.version;
} else { } else {
@ -2389,7 +2391,7 @@ const char* tmq_get_table_name(TAOS_RES* res) {
return NULL; return NULL;
} }
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res;
if (!common->withTbName || common->blockTbName == NULL || pRspObj->resIter < 0 || if (!common->withTbName || common->blockTbName == NULL || pRspObj->resIter < 0 ||
@ -2734,8 +2736,8 @@ void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, cons
} }
SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res;
pRspObj->resIter++; pRspObj->resIter++;
if (pRspObj->resIter < common->blockNum) { if (pRspObj->resIter < common->blockNum) {
if (common->withSchema) { if (common->withSchema) {
@ -2744,8 +2746,8 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
} }
void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter); void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter);
void* rawData = NULL; void* rawData = NULL;
int64_t rows = 0; int64_t rows = 0;
int32_t precision = 0; int32_t precision = 0;
tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, &precision); tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, &precision);

26
source/common/src/cos.c
View File

@ -83,10 +83,10 @@ int32_t s3CheckCfg() {
snprintf(path, PATH_MAX, "%s", tsTempDir); snprintf(path, PATH_MAX, "%s", tsTempDir);
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
snprintf(path + tmp_len, PATH_MAX, "%s", TD_DIRSEP); snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
snprintf(path + tmp_len + ds_len, PATH_MAX, "%s", objectname[0]); snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", objectname[0]);
} else { } else {
snprintf(path + tmp_len, PATH_MAX, "%s", objectname[0]); snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", objectname[0]);
} }
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
@ -1075,32 +1075,32 @@ static SArray *getListByPrefix(const char *prefix) {
S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
&listBucketCallback}; &listBucketCallback};
const char *marker = 0, *delimiter = 0; const char /**marker = 0,*/ *delimiter = 0;
int maxkeys = 0, allDetails = 0; int /*maxkeys = 0, */ allDetails = 0;
list_bucket_callback_data data = {0}; list_bucket_callback_data data = {0};
data.objectArray = taosArrayInit(32, sizeof(void *)); data.objectArray = taosArrayInit(32, sizeof(void *));
if (!data.objectArray) { if (!data.objectArray) {
uError("%s: %s", __func__, "out of memoty"); uError("%s: %s", __func__, "out of memoty");
return NULL; return NULL;
} }
if (marker) { /*if (marker) {
snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker);
} else { } else {*/
data.nextMarker[0] = 0; data.nextMarker[0] = 0;
} //}
data.keyCount = 0; data.keyCount = 0;
data.allDetails = allDetails; data.allDetails = allDetails;
do { do {
data.isTruncated = 0; data.isTruncated = 0;
do { do {
S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler, S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, 0 /*maxkeys*/, 0, timeoutMsG,
&data); &listBucketHandler, &data);
} while (S3_status_is_retryable(data.status) && should_retry()); } while (S3_status_is_retryable(data.status) && should_retry());
if (data.status != S3StatusOK) { if (data.status != S3StatusOK) {
break; break;
} }
} while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); } while (data.isTruncated /* && (!maxkeys || (data.keyCount < maxkeys))*/);
if (data.status == S3StatusOK) { if (data.status == S3StatusOK) {
if (data.keyCount > 0) { if (data.keyCount > 0) {

9
source/common/src/cos_cp.c
View File

@ -20,7 +20,7 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
} }
void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); } void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); }
void cos_cp_remove(char const* filepath) { taosRemoveFile(filepath); } void cos_cp_remove(char const* filepath) { (void)taosRemoveFile(filepath); }
static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) { static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
int32_t code = 0; int32_t code = 0;
@ -380,7 +380,7 @@ void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPa
void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64) { void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64) {
checkpoint->parts[part_index].completed = 1; checkpoint->parts[part_index].completed = 1;
strncpy(checkpoint->parts[part_index].etag, etag, 128); strncpy(checkpoint->parts[part_index].etag, etag, 127);
checkpoint->parts[part_index].crc64 = crc64; checkpoint->parts[part_index].crc64 = crc64;
} }
@ -389,11 +389,12 @@ void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t
int i = 0; int i = 0;
checkpoint->cp_type = COS_CP_TYPE_UPLOAD; checkpoint->cp_type = COS_CP_TYPE_UPLOAD;
strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN); memset(checkpoint->file_path, 0, TSDB_FILENAME_LEN);
strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN - 1);
checkpoint->file_size = size; checkpoint->file_size = size;
checkpoint->file_last_modified = mtime; checkpoint->file_last_modified = mtime;
strncpy(checkpoint->upload_id, upload_id, 128); strncpy(checkpoint->upload_id, upload_id, 127);
checkpoint->part_size = part_size; checkpoint->part_size = part_size;
for (; i * part_size < size; i++) { for (; i * part_size < size; i++) {

67
source/common/src/rsync.c
View File

@ -5,8 +5,8 @@
#include <stdlib.h> #include <stdlib.h>
#include "tglobal.h" #include "tglobal.h"
#define ERRNO_ERR_FORMAT "errno:%d,msg:%s" #define ERRNO_ERR_FORMAT "errno:%d,msg:%s"
#define ERRNO_ERR_DATA errno,strerror(errno) #define ERRNO_ERR_DATA errno, strerror(errno)
// deleteRsync function produce empty directories, traverse base directory to remove them // deleteRsync function produce empty directories, traverse base directory to remove them
static void removeEmptyDir() { static void removeEmptyDir() {
@ -24,14 +24,14 @@ static void removeEmptyDir() {
char filename[PATH_MAX] = {0}; char filename[PATH_MAX] = {0};
snprintf(filename, sizeof(filename), "%s%s", tsCheckpointBackupDir, taosGetDirEntryName(de)); snprintf(filename, sizeof(filename), "%s%s", tsCheckpointBackupDir, taosGetDirEntryName(de));
TdDirPtr pDirTmp = taosOpenDir(filename); TdDirPtr pDirTmp = taosOpenDir(filename);
TdDirEntryPtr deTmp = NULL; TdDirEntryPtr deTmp = NULL;
bool empty = true; bool empty = true;
while ((deTmp = taosReadDir(pDirTmp)) != NULL){ while ((deTmp = taosReadDir(pDirTmp)) != NULL) {
if (strcmp(taosGetDirEntryName(deTmp), ".") == 0 || strcmp(taosGetDirEntryName(deTmp), "..") == 0) continue; if (strcmp(taosGetDirEntryName(deTmp), ".") == 0 || strcmp(taosGetDirEntryName(deTmp), "..") == 0) continue;
empty = false; empty = false;
} }
if(empty) taosRemoveDir(filename); if (empty) taosRemoveDir(filename);
taosCloseDir(&pDirTmp); taosCloseDir(&pDirTmp);
} }
@ -40,10 +40,10 @@ static void removeEmptyDir() {
#ifdef WINDOWS #ifdef WINDOWS
// C:\TDengine\data\backup\checkpoint\ -> /c/TDengine/data/backup/checkpoint/ // C:\TDengine\data\backup\checkpoint\ -> /c/TDengine/data/backup/checkpoint/
static void changeDirFromWindowsToLinux(char* from, char* to){ static void changeDirFromWindowsToLinux(char* from, char* to) {
to[0] = '/'; to[0] = '/';
to[1] = from[0]; to[1] = from[0];
for(int32_t i = 2; i < strlen(from); i++) { for (int32_t i = 2; i < strlen(from); i++) {
if (from[i] == '\\') { if (from[i] == '\\') {
to[i] = '/'; to[i] = '/';
} else { } else {
@ -56,7 +56,7 @@ static void changeDirFromWindowsToLinux(char* from, char* to){
static int32_t generateConfigFile(char* confDir) { static int32_t generateConfigFile(char* confDir) {
TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) { if (pFile == NULL) {
uError("[rsync] open conf file error, dir:%s,"ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA); uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA);
return -1; return -1;
} }
@ -65,8 +65,8 @@ static int32_t generateConfigFile(char* confDir) {
changeDirFromWindowsToLinux(tsCheckpointBackupDir, path); changeDirFromWindowsToLinux(tsCheckpointBackupDir, path);
#endif #endif
char confContent[PATH_MAX*4] = {0}; char confContent[PATH_MAX * 4] = {0};
snprintf(confContent, PATH_MAX*4, snprintf(confContent, PATH_MAX * 4,
#ifndef WINDOWS #ifndef WINDOWS
"uid = root\n" "uid = root\n"
"gid = root\n" "gid = root\n"
@ -80,16 +80,17 @@ static int32_t generateConfigFile(char* confDir) {
"read only = false\n" "read only = false\n"
"list = false\n" "list = false\n"
"[checkpoint]\n" "[checkpoint]\n"
"path = %s", tsCheckpointBackupDir, tsCheckpointBackupDir, "path = %s",
tsCheckpointBackupDir, tsCheckpointBackupDir,
#ifdef WINDOWS #ifdef WINDOWS
path path
#else #else
tsCheckpointBackupDir tsCheckpointBackupDir
#endif #endif
); );
uDebug("[rsync] conf:%s", confContent); uDebug("[rsync] conf:%s", confContent);
if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0){ if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) {
uError("[rsync] write conf file error,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
taosCloseFile(&pFile); taosCloseFile(&pFile);
return -1; return -1;
} }
@ -98,10 +99,10 @@ static int32_t generateConfigFile(char* confDir) {
return 0; return 0;
} }
static int32_t execCommand(char* command){ static int32_t execCommand(char* command) {
int32_t try = 3; int32_t try = 3;
int32_t code = 0; int32_t code = 0;
while(try-- > 0) { while (try-- > 0) {
code = system(command); code = system(command);
if (code == 0) { if (code == 0) {
break; break;
@ -125,7 +126,7 @@ void stopRsync() {
uDebug("[rsync] stop rsync server successful"); uDebug("[rsync] stop rsync server successful");
} }
taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation. taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation.
} }
void startRsync() { void startRsync() {
@ -154,7 +155,6 @@ void startRsync() {
} else { } else {
uDebug("[rsync] start server successful"); uDebug("[rsync] start server successful");
} }
} }
int32_t uploadByRsync(const char* id, const char* path) { int32_t uploadByRsync(const char* id, const char* path) {
@ -165,7 +165,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
char pathTransform[PATH_MAX] = {0}; char pathTransform[PATH_MAX] = {0};
changeDirFromWindowsToLinux(path, pathTransform); changeDirFromWindowsToLinux(path, pathTransform);
if(pathTransform[strlen(pathTransform) - 1] != '/') { if (pathTransform[strlen(pathTransform) - 1] != '/') {
#else #else
if (path[strlen(path) - 1] != '/') { if (path[strlen(path) - 1] != '/') {
#endif #endif
@ -197,12 +197,12 @@ int32_t uploadByRsync(const char* id, const char* path) {
// prepare the data directory // prepare the data directory
int32_t code = execCommand(command); int32_t code = execCommand(command);
if (code != 0) { if (code != 0) {
uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA); tsSnodeAddress, code, ERRNO_ERR_DATA);
} else { } else {
int64_t el = (taosGetTimestampMs() - st); int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
tsSnodeAddress, el); tsSnodeAddress, el);
} }
#ifdef WINDOWS #ifdef WINDOWS
@ -264,19 +264,20 @@ int32_t downloadRsync(const char* id, const char* path) {
#endif #endif
char command[PATH_MAX] = {0}; char command[PATH_MAX] = {0};
snprintf(command, PATH_MAX, snprintf(
"rsync -av --debug=all --log-file=%s/rsynclog --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/data/ %s", command, PATH_MAX,
tsLogDir, tsSnodeAddress, id, "rsync -av --debug=all --log-file=%s/rsynclog --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/data/ %s",
tsLogDir, tsSnodeAddress, id,
#ifdef WINDOWS #ifdef WINDOWS
pathTransform pathTransform
#else #else
path path
#endif #endif
); );
uDebug("[rsync] %s start to sync data from remote to:%s, %s", id, path, command); uDebug("[rsync] %s start to sync data from remote to:%s, %s", id, path, command);
while(times++ < MAX_RETRY) { while (times++ < MAX_RETRY) {
code = execCommand(command); code = execCommand(command);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id, uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id,
@ -302,8 +303,8 @@ int32_t deleteRsync(const char* id) {
char command[PATH_MAX] = {0}; char command[PATH_MAX] = {0};
snprintf(command, PATH_MAX, snprintf(command, PATH_MAX,
"rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 %s rsync://%s/checkpoint/%s/data/", tsLogDir, "rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 %s rsync://%s/checkpoint/%s/data/",
tmp, tsSnodeAddress, id); tsLogDir, tmp, tsSnodeAddress, id);
code = execCommand(command); code = execCommand(command);
taosRemoveDir(tmp); taosRemoveDir(tmp);

2
source/common/src/tdataformat.c
View File

@ -2983,7 +2983,7 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt
} }
} else { } else {
if (varDataTLen(data + offset) > bytes) { if (varDataTLen(data + offset) > bytes) {
uError("var data length invalid, varDataTLen(data + offset):%d >= bytes:%d", (int)varDataTLen(data + offset), uError("var data length invalid, varDataTLen(data + offset):%d > bytes:%d", (int)varDataTLen(data + offset),
bytes); bytes);
code = TSDB_CODE_PAR_VALUE_TOO_LONG; code = TSDB_CODE_PAR_VALUE_TOO_LONG;
goto _exit; goto _exit;

4
source/common/src/tglobal.c
View File

@ -392,9 +392,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
return 0; return 0;
} }
struct SConfig *taosGetCfg() { struct SConfig *taosGetCfg() { return tsCfg; }
return tsCfg;
}
static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
char *apolloUrl) { char *apolloUrl) {

5
source/common/src/tmsg.c
View File

@ -10522,7 +10522,10 @@ int32_t tCloneTbTSMAInfo(STableTSMAInfo *pInfo, STableTSMAInfo **pRes) {
pRet->ast = taosStrdup(pInfo->ast); pRet->ast = taosStrdup(pInfo->ast);
if (!pRet->ast) code = TSDB_CODE_OUT_OF_MEMORY; if (!pRet->ast) code = TSDB_CODE_OUT_OF_MEMORY;
} }
if (code) tFreeTableTSMAInfo(pRet); if (code) {
tFreeTableTSMAInfo(pRet);
pRet = NULL;
}
*pRes = pRet; *pRes = pRet;
return code; return code;
} }

21
source/common/src/tvariant.c
View File

@ -161,8 +161,7 @@ int32_t toIntegerPure(const char *z, int32_t n, int32_t base, int64_t *value) {
int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
errno = 0; errno = 0;
char *endPtr = NULL; char *endPtr = NULL;
switch (type) switch (type) {
{
case TK_NK_INTEGER: { case TK_NK_INTEGER: {
*value = taosStr2Int64(z, &endPtr, 10); *value = taosStr2Int64(z, &endPtr, 10);
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
@ -172,7 +171,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
} break; } break;
case TK_NK_FLOAT: { case TK_NK_FLOAT: {
double val = round(taosStr2Double(z, &endPtr)); double val = round(taosStr2Double(z, &endPtr));
if(val < (double)INT64_MIN || val > (double)INT64_MAX){ if (val < (double)INT64_MIN || val > (double)INT64_MAX) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
@ -181,8 +180,8 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
*value = val; *value = val;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} break; } break;
default: default:
break; break;
} }
if (n == 0) { if (n == 0) {
@ -201,7 +200,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
// pure decimal part // pure decimal part
const char *s = endPtr + 1; const char *s = endPtr + 1;
const char *end = z + n; const char *end = z + n;
bool pure = true; bool pure = true;
while (s < end) { while (s < end) {
if (*s < '0' || *s > '9') { if (*s < '0' || *s > '9') {
pure = false; pure = false;
@ -210,17 +209,17 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
s++; s++;
} }
if (pure) { if (pure) {
if (endPtr+1 < end && endPtr[1] > '4') { if (endPtr + 1 < end && endPtr[1] > '4') {
const char *p = z; const char *p = z;
while (*p == ' ') { while (*p == ' ') {
p++; p++;
} }
if (*p == '-') { if (*p == '-') {
if ( *value > INT64_MIN) { if (*value > INT64_MIN) {
(*value)--; (*value)--;
} }
} else { } else {
if ( *value < INT64_MAX) { if (*value < INT64_MAX) {
(*value)++; (*value)++;
} }
} }
@ -229,7 +228,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
} }
} }
// 2. parse as other // 2. parse as other
bool is_neg = false; bool is_neg = false;
uint64_t uv = 0; uint64_t uv = 0;
int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv, true); int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv, true);
@ -256,7 +255,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
errno = 0; errno = 0;
char *endPtr = NULL; char *endPtr = NULL;
const char *p = z; const char *p = z;
switch (type) { switch (type) {
case TK_NK_INTEGER: { case TK_NK_INTEGER: {

9
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -53,7 +53,8 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
.msgType = TDMT_MND_RETRIEVE_IP_WHITE, .msgType = TDMT_MND_RETRIEVE_IP_WHITE,
.info.ahandle = (void *)0x9527, .info.ahandle = (void *)0x9527,
.info.refId = 0, .info.refId = 0,
.info.noResp = 0}; .info.noResp = 0,
.info.handle = 0};
SEpSet epset = {0}; SEpSet epset = {0};
dmGetMnodeEpSet(pMgmt->pData, &epset); dmGetMnodeEpSet(pMgmt->pData, &epset);
@ -154,7 +155,8 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
.msgType = TDMT_MND_STATUS, .msgType = TDMT_MND_STATUS,
.info.ahandle = (void *)0x9527, .info.ahandle = (void *)0x9527,
.info.refId = 0, .info.refId = 0,
.info.noResp = 0}; .info.noResp = 0,
.info.handle = 0};
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq); dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq);
@ -186,7 +188,8 @@ void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) {
.msgType = TDMT_MND_NOTIFY, .msgType = TDMT_MND_NOTIFY,
.info.ahandle = (void *)0x9527, .info.ahandle = (void *)0x9527,
.info.refId = 0, .info.refId = 0,
.info.noResp = 1}; .info.noResp = 1,
.info.handle = 0};
SEpSet epSet = {0}; SEpSet epSet = {0};
dmGetMnodeEpSet(pMgmt->pData, &epSet); dmGetMnodeEpSet(pMgmt->pData, &epSet);

1
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -290,6 +290,7 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle);
return -1; return -1;
} else { } else {
pMsg->info.handle = 0;
rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL); rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL);
return 0; return 0;
} }

103
source/dnode/mnode/impl/src/mndStream.c
View File

@ -56,7 +56,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int
int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger); int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger);
static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList); static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList);
static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq); static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq);
static int32_t mndProcessCheckpointReport(SRpcMsg *pReq); static int32_t mndProcessCheckpointReport(SRpcMsg *pReq);
@ -301,17 +301,17 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) {
return 0; return 0;
} }
static int32_t createSchemaByFields(const SArray* pFields, SSchemaWrapper* pWrapper) { static int32_t createSchemaByFields(const SArray *pFields, SSchemaWrapper *pWrapper) {
pWrapper->nCols = taosArrayGetSize(pFields); pWrapper->nCols = taosArrayGetSize(pFields);
pWrapper->pSchema = taosMemoryCalloc(pWrapper->nCols, sizeof(SSchema)); pWrapper->pSchema = taosMemoryCalloc(pWrapper->nCols, sizeof(SSchema));
if (NULL == pWrapper->pSchema) { if (NULL == pWrapper->pSchema) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
SNode* pNode; SNode *pNode;
int32_t index = 0; int32_t index = 0;
for(int32_t i = 0; i < pWrapper->nCols; i++) { for (int32_t i = 0; i < pWrapper->nCols; i++) {
SField* pField = (SField*)taosArrayGet(pFields, i); SField *pField = (SField *)taosArrayGet(pFields, i);
if (TSDB_DATA_TYPE_NULL == pField->type) { if (TSDB_DATA_TYPE_NULL == pField->type) {
pWrapper->pSchema[index].type = TSDB_DATA_TYPE_VARCHAR; pWrapper->pSchema[index].type = TSDB_DATA_TYPE_VARCHAR;
pWrapper->pSchema[index].bytes = VARSTR_HEADER_SIZE; pWrapper->pSchema[index].bytes = VARSTR_HEADER_SIZE;
@ -328,14 +328,14 @@ static int32_t createSchemaByFields(const SArray* pFields, SSchemaWrapper* pWrap
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool hasDestPrimaryKey(SSchemaWrapper* pWrapper) { static bool hasDestPrimaryKey(SSchemaWrapper *pWrapper) {
if (pWrapper->nCols < 2) { if (pWrapper->nCols < 2) {
return false; return false;
} }
for (int32_t i = 1; i < pWrapper->nCols; i++) { for (int32_t i = 1; i < pWrapper->nCols; i++) {
if(pWrapper->pSchema[i].flags & COL_IS_KEY) { if (pWrapper->pSchema[i].flags & COL_IS_KEY) {
return true; return true;
} }
} }
return false; return false;
} }
@ -445,7 +445,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
pObj->outputSchema.pSchema = pFullSchema; pObj->outputSchema.pSchema = pFullSchema;
} }
bool hasKey = hasDestPrimaryKey(&pObj->outputSchema); bool hasKey = hasDestPrimaryKey(&pObj->outputSchema);
SPlanContext cxt = { SPlanContext cxt = {
.pAstRoot = pAst, .pAstRoot = pAst,
.topicQuery = false, .topicQuery = false,
@ -492,7 +492,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, NULL, 0); tEncoderInit(&encoder, NULL, 0);
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) {
pTask->ver = SSTREAM_TASK_VER; pTask->ver = SSTREAM_TASK_VER;
} }
tEncodeStreamTask(&encoder, pTask); tEncodeStreamTask(&encoder, pTask);
@ -700,7 +700,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
SStreamObj streamObj = {0}; SStreamObj streamObj = {0};
char *sql = NULL; char *sql = NULL;
int32_t sqlLen = 0; int32_t sqlLen = 0;
const char* pMsg = "create stream tasks on dnodes"; const char *pMsg = "create stream tasks on dnodes";
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
@ -959,7 +959,8 @@ static int32_t doSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask
return -1; return -1;
} }
code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0); code =
setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(buf); taosMemoryFree(buf);
} }
@ -1106,7 +1107,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0); ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0);
} }
SArray* pInvalidList = taosArrayInit(4, sizeof(STaskId)); SArray *pInvalidList = taosArrayInit(4, sizeof(STaskId));
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskId *p = taosArrayGet(execInfo.pTaskList, i);
@ -1116,8 +1117,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
} }
if (pEntry->status == TASK_STATUS__STOP) { if (pEntry->status == TASK_STATUS__STOP) {
for(int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) { for (int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) {
STaskId* pId = taosArrayGet(pInvalidList, j); STaskId *pId = taosArrayGet(pInvalidList, j);
if (pEntry->id.streamId == pId->streamId) { if (pEntry->id.streamId == pId->streamId) {
taosArrayPush(pInvalidList, &pEntry->id); taosArrayPush(pInvalidList, &pEntry->id);
break; break;
@ -1126,8 +1127,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
} }
if (pEntry->status != TASK_STATUS__READY) { if (pEntry->status != TASK_STATUS__READY) {
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued", mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued", pEntry->id.streamId,
pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status)); (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status));
ready = false; ready = false;
} }
@ -1153,14 +1154,14 @@ typedef struct {
int64_t duration; int64_t duration;
} SCheckpointInterval; } SCheckpointInterval;
static int32_t streamWaitComparFn(const void* p1, const void* p2) { static int32_t streamWaitComparFn(const void *p1, const void *p2) {
const SCheckpointInterval* pInt1 = p1; const SCheckpointInterval *pInt1 = p1;
const SCheckpointInterval* pInt2 = p2; const SCheckpointInterval *pInt2 = p2;
if (pInt1->duration == pInt2->duration) { if (pInt1->duration == pInt2->duration) {
return 0; return 0;
} }
return pInt1->duration > pInt2->duration? -1:1; return pInt1->duration > pInt2->duration ? -1 : 1;
} }
static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
@ -1176,7 +1177,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
return -1; return -1;
} }
SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval)); SArray *pList = taosArrayInit(4, sizeof(SCheckpointInterval));
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) {
@ -1190,8 +1191,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
taosArrayPush(pList, &in); taosArrayPush(pList, &in);
int32_t currentSize = taosArrayGetSize(pList); int32_t currentSize = taosArrayGetSize(pList);
mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64 mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64 "s) beyond threshold:%d",
"s) beyond threshold:%d",
pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize); pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize);
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
@ -1278,11 +1278,11 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
} }
if (pStream->smaId != 0) { if (pStream->smaId != 0) {
mDebug("stream:%s, uid:0x%"PRIx64" try to drop sma related stream", dropReq.name, pStream->uid); mDebug("stream:%s, uid:0x%" PRIx64 " try to drop sma related stream", dropReq.name, pStream->uid);
void *pIter = NULL; void *pIter = NULL;
SSmaObj *pSma = NULL; SSmaObj *pSma = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void**)&pSma); pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma);
while (pIter) { while (pIter) {
if (pSma && pSma->uid == pStream->smaId) { if (pSma && pSma->uid == pStream->smaId) {
sdbRelease(pMnode->pSdb, pSma); sdbRelease(pMnode->pSdb, pSma);
@ -1321,7 +1321,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s uid:0x%"PRIx64" failed to drop since %s", dropReq.name, pStream->uid, terrstr()); mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq); tFreeMDropStreamReq(&dropReq);
return -1; return -1;
@ -1674,8 +1674,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
// input total // input total
const char* formatTotalMb = "%7.2f MiB"; const char *formatTotalMb = "%7.2f MiB";
const char* formatTotalGb = "%7.2f GiB"; const char *formatTotalGb = "%7.2f GiB";
if (pe->procsTotal < 1024) { if (pe->procsTotal < 1024) {
snprintf(buf, tListLen(buf), formatTotalMb, pe->procsTotal); snprintf(buf, tListLen(buf), formatTotalMb, pe->procsTotal);
} else { } else {
@ -1689,8 +1689,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
// process throughput // process throughput
const char* formatKb = "%7.2f KiB/s"; const char *formatKb = "%7.2f KiB/s";
const char* formatMb = "%7.2f MiB/s"; const char *formatMb = "%7.2f MiB/s";
if (pe->procsThroughput < 1024) { if (pe->procsThroughput < 1024) {
snprintf(buf, tListLen(buf), formatKb, pe->procsThroughput); snprintf(buf, tListLen(buf), formatKb, pe->procsThroughput);
} else { } else {
@ -1760,15 +1760,15 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
// start_time // start_time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startTime, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startTime, false);
// start id // start id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startCheckpointId, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointId, false);
// start ver // start ver
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startCheckpointVer, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointVer, false);
// checkpoint time // checkpoint time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@ -1780,11 +1780,11 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
// checkpoint_id // checkpoint_id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestId, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestId, false);
// checkpoint version // checkpoint version
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestVer, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestVer, false);
// checkpoint size // checkpoint size
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@ -1915,13 +1915,12 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
{ // check for tasks, if tasks are not ready, not allowed to pause
{ // check for tasks, if tasks are not ready, not allowed to pause
bool found = false; bool found = false;
bool readyToPause = true; bool readyToPause = true;
taosThreadMutexLock(&execInfo.lock); taosThreadMutexLock(&execInfo.lock);
for(int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
@ -2237,7 +2236,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
return 0; return 0;
} }
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) { static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -2359,7 +2358,7 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
return 0; return 0;
} }
int32_t size = sizeof(SMStreamNodeCheckMsg); int32_t size = sizeof(SMStreamNodeCheckMsg);
SMStreamNodeCheckMsg *pMsg = rpcMallocCont(size); SMStreamNodeCheckMsg *pMsg = rpcMallocCont(size);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = size}; SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = size};
@ -2386,8 +2385,8 @@ void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode)
// add the new vgroups if not added yet // add the new vgroups if not added yet
bool exist = false; bool exist = false;
for(int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) { for (int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) {
SNodeEntry* pEntry = taosArrayGet(pExecNode->pNodeList, j); SNodeEntry *pEntry = taosArrayGet(pExecNode->pNodeList, j);
if (pEntry->nodeId == pTask->info.nodeId) { if (pEntry->nodeId == pTask->info.nodeId) {
exist = true; exist = true;
break; break;
@ -2514,10 +2513,10 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
return 0; return 0;
} }
static void doAddTaskInfo(SArray* pList, SCheckpointReport* pReport) { static void doAddTaskInfo(SArray *pList, SCheckpointReport *pReport) {
bool existed = false; bool existed = false;
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
STaskChkptInfo* p = taosArrayGet(pList ,i); STaskChkptInfo *p = taosArrayGet(pList, i);
if (p->taskId == pReport->taskId) { if (p->taskId == pReport->taskId) {
existed = true; existed = true;
break; break;
@ -2563,8 +2562,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
if (pStream == NULL) { if (pStream == NULL) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId);
req.streamId);
// not in meta-store yet, try to acquire the task in exec buffer // not in meta-store yet, try to acquire the task in exec buffer
// the checkpoint req arrives too soon before the completion of the create stream trans. // the checkpoint req arrives too soon before the completion of the create stream trans.
@ -2671,7 +2669,7 @@ void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
pExecInfo->initTaskList = true; pExecInfo->initTaskList = true;
} }
void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) { void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -2687,13 +2685,14 @@ void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) {
} }
} }
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray* pChkptInfoList) { int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, "update checkpoint-info"); STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME,
"update checkpoint-info");
if (pTrans == NULL) { if (pTrans == NULL) {
return terrno; return terrno;
} }
/*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid); /*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid);
int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream); int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream);
if (code != 0) { if (code != 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);

36
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -39,10 +39,10 @@ static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) {
} }
} }
static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pInfo) { static void addIntoCheckpointList(SArray *pList, const SFailedCheckpointInfo *pInfo) {
int32_t num = taosArrayGetSize(pList); int32_t num = taosArrayGetSize(pList);
for(int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
SFailedCheckpointInfo* p = taosArrayGet(pList, i); SFailedCheckpointInfo *p = taosArrayGet(pList, i);
if (p->transId == pInfo->transId) { if (p->transId == pInfo->transId) {
return; return;
} }
@ -52,12 +52,13 @@ static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pI
} }
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint"); STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME,
" reset from failed checkpoint");
if (pTrans == NULL) { if (pTrans == NULL) {
return terrno; return terrno;
} }
/*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); /*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid);
int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream); int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream);
if (code != 0) { if (code != 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -140,8 +141,8 @@ static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) { static int32_t mndDropOrphanTasks(SMnode *pMnode, SArray *pList) {
SOrphanTask* pTask = taosArrayGet(pList, 0); SOrphanTask *pTask = taosArrayGet(pList, 0);
// check if it is conflict with other trans in both sourceDb and targetDb. // check if it is conflict with other trans in both sourceDb and targetDb.
bool conflict = mndStreamTransConflictCheck(pMnode, pTask->streamId, MND_STREAM_DROP_NAME, false); bool conflict = mndStreamTransConflictCheck(pMnode, pTask->streamId, MND_STREAM_DROP_NAME, false);
@ -150,7 +151,7 @@ static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) {
} }
SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""}; SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""};
STrans* pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); STrans *pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("failed to create trans to drop orphan tasks since %s", terrstr()); mError("failed to create trans to drop orphan tasks since %s", terrstr());
return -1; return -1;
@ -180,21 +181,21 @@ static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) {
return 0; return 0;
} }
int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void* pIter = NULL; void *pIter = NULL;
while(1) { while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break; if (pIter == NULL) break;
if(pStream->status != STREAM_STATUS__PAUSE){ if (pStream->status != STREAM_STATUS__PAUSE) {
SMPauseStreamReq reqPause = {0}; SMPauseStreamReq reqPause = {0};
strcpy(reqPause.name, pStream->name); strcpy(reqPause.name, pStream->name);
reqPause.igNotExists = 1; reqPause.igNotExists = 1;
int32_t contLen = tSerializeSMPauseStreamReq(NULL, 0, &reqPause); int32_t contLen = tSerializeSMPauseStreamReq(NULL, 0, &reqPause);
void * pHead = rpcMallocCont(contLen); void *pHead = rpcMallocCont(contLen);
tSerializeSMPauseStreamReq(pHead, contLen, &reqPause); tSerializeSMPauseStreamReq(pHead, contLen, &reqPause);
SRpcMsg rpcMsg = { SRpcMsg rpcMsg = {
@ -205,7 +206,8 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){
}; };
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
mInfo("receive pause stream:%s, %s, %"PRId64 ", because grant expired", pStream->name, reqPause.name, pStream->uid); mInfo("receive pause stream:%s, %s, %" PRId64 ", because grant expired", pStream->name, reqPause.name,
pStream->uid);
} }
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
@ -311,7 +313,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
if (allReady || snodeChanged) { if (allReady || snodeChanged) {
// if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal
for(int32_t i = 0; i < taosArrayGetSize(pFailedChkpt); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pFailedChkpt); ++i) {
SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedChkpt, i); SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedChkpt, i);
mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status", mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status",
pInfo->checkpointId, pInfo->transId); pInfo->checkpointId, pInfo->transId);
@ -341,11 +343,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
{ {
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamHbRspMsg)}; SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamHbRspMsg)};
rsp.pCont = rpcMallocCont(rsp.contLen); rsp.pCont = rpcMallocCont(rsp.contLen);
SMsgHead* pHead = rsp.pCont; SMsgHead *pHead = rsp.pCont;
pHead->vgId = htonl(req.vgId); pHead->vgId = htonl(req.vgId);
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
pReq->info.handle = NULL; // disable auto rsp pReq->info.handle = NULL; // disable auto rsp
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

36
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -17,21 +17,21 @@
#include "mndTrans.h" #include "mndTrans.h"
typedef struct SKeyInfo { typedef struct SKeyInfo {
void* pKey; void *pKey;
int32_t keyLen; int32_t keyLen;
} SKeyInfo; } SKeyInfo;
int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamId) { int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId) {
SStreamTransInfo info = { SStreamTransInfo info = {
.transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamId = streamId}; .transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamId = streamId};
taosHashPut(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId), &info, sizeof(SStreamTransInfo)); taosHashPut(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId), &info, sizeof(SStreamTransInfo));
return 0; return 0;
} }
int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt) { int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) {
size_t keyLen = 0; size_t keyLen = 0;
void* pIter = NULL; void *pIter = NULL;
SArray* pList = taosArrayInit(4, sizeof(SKeyInfo)); SArray *pList = taosArrayInit(4, sizeof(SKeyInfo));
int32_t num = 0; int32_t num = 0;
while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) {
@ -43,7 +43,8 @@ int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt)
void *pKey = taosHashGetKey(pEntry, &keyLen); void *pKey = taosHashGetKey(pEntry, &keyLen);
// key is the name of src/dst db name // key is the name of src/dst db name
SKeyInfo info = {.pKey = pKey, .keyLen = keyLen}; SKeyInfo info = {.pKey = pKey, .keyLen = keyLen};
mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, pEntry->startTime); mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name,
pEntry->startTime);
taosArrayPush(pList, &info); taosArrayPush(pList, &info);
} else { } else {
if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) { if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) {
@ -55,7 +56,7 @@ int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt)
int32_t size = taosArrayGetSize(pList); int32_t size = taosArrayGetSize(pList);
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SKeyInfo* pKey = taosArrayGet(pList, i); SKeyInfo *pKey = taosArrayGet(pList, i);
taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen); taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen);
} }
@ -76,7 +77,7 @@ int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt)
// For a given stream: // For a given stream:
// 1. checkpoint trans is conflict with any other trans except for the drop and reset trans. // 1. checkpoint trans is conflict with any other trans except for the drop and reset trans.
// 2. create/drop/reset/update trans are conflict with any other trans. // 2. create/drop/reset/update trans are conflict with any other trans.
bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* pTransName, bool lock) { bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock) {
if (lock) { if (lock) {
taosThreadMutexLock(&execInfo.lock); taosThreadMutexLock(&execInfo.lock);
} }
@ -127,7 +128,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p
return false; return false;
} }
int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) { int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) {
taosThreadMutexLock(&execInfo.lock); taosThreadMutexLock(&execInfo.lock);
int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans);
if (num <= 0) { if (num <= 0) {
@ -136,7 +137,7 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) {
} }
mndStreamClearFinishedTrans(pMnode, NULL); mndStreamClearFinishedTrans(pMnode, NULL);
SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId)); SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
if (pEntry != NULL) { if (pEntry != NULL) {
SStreamTransInfo tInfo = *pEntry; SStreamTransInfo tInfo = *pEntry;
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
@ -152,7 +153,8 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) {
return 0; return 0;
} }
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg) { STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
@ -208,7 +210,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
STREAM_ENCODE_OVER: STREAM_ENCODE_OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr()); mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr());
@ -248,12 +250,16 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status)
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode, int32_t acceptCode) { int32_t retryCode, int32_t acceptCode) {
STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode, STransAction action = {.epSet = *pEpset,
.contLen = contLen,
.pCont = pCont,
.msgType = msgType,
.retryCode = retryCode,
.acceptableCode = acceptCode}; .acceptableCode = acceptCode};
return mndTransAppendRedoAction(pTrans, &action); return mndTransAppendRedoAction(pTrans, &action);
} }
static bool identicalName(const char* pDb, const char* pParam, int32_t len) { static bool identicalName(const char *pDb, const char *pParam, int32_t len) {
return (strlen(pDb) == len) && (strncmp(pDb, pParam, len) == 0); return (strlen(pDb) == len) && (strncmp(pDb, pParam, len) == 0);
} }
@ -300,5 +306,3 @@ void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) {
mDebug("complete clear checkpoints in Dbs"); mDebug("complete clear checkpoints in Dbs");
} }

265
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -41,7 +41,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg);
static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg); static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg);
static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter);
static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* hash); static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash);
static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) { static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) {
SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); SSdbRaw *pCommitRaw = mndSubActionEncode(pSub);
@ -95,13 +95,13 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
return pSub; return pSub;
} }
static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg,
const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { SSubplan *pPlan) {
SMqRebVgReq req = {0}; SMqRebVgReq req = {0};
req.oldConsumerId = pRebVg->oldConsumerId; req.oldConsumerId = pRebVg->oldConsumerId;
req.newConsumerId = pRebVg->newConsumerId; req.newConsumerId = pRebVg->newConsumerId;
req.vgId = pRebVg->pVgEp->vgId; req.vgId = pRebVg->pVgEp->vgId;
if(pPlan){ if (pPlan) {
pPlan->execNode.epSet = pRebVg->pVgEp->epSet; pPlan->execNode.epSet = pRebVg->pVgEp->epSet;
pPlan->execNode.nodeId = pRebVg->pVgEp->vgId; pPlan->execNode.nodeId = pRebVg->pVgEp->vgId;
int32_t msgLen; int32_t msgLen;
@ -109,7 +109,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
terrno = TSDB_CODE_QRY_INVALID_INPUT; terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1; return -1;
} }
}else{ } else {
req.qmsg = taosStrdup(""); req.qmsg = taosStrdup("");
} }
req.subType = pSub->subType; req.subType = pSub->subType;
@ -126,7 +126,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
} }
tlen += sizeof(SMsgHead); tlen += sizeof(SMsgHead);
void *buf = taosMemoryMalloc(tlen); void *buf = taosMemoryMalloc(tlen);
if (buf == NULL) { if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(req.qmsg); taosMemoryFree(req.qmsg);
@ -155,9 +155,9 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
} }
static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub,
const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { const SMqRebOutputVg *pRebVg, SSubplan *pPlan) {
if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { if (pRebVg->oldConsumerId == pRebVg->newConsumerId) {
if(pRebVg->oldConsumerId == -1) return 0; //drop stream, no consumer, while split vnode,all consumerId is -1 if (pRebVg->oldConsumerId == -1) return 0; // drop stream, no consumer, while split vnode,all consumerId is -1
terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; terrno = TSDB_CODE_MND_INVALID_SUB_OPTION;
return -1; return -1;
} }
@ -223,19 +223,18 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
return pRebInfo; return pRebInfo;
} }
static void pushVgDataToHash(SArray *vgs, SHashObj *pHash, int64_t consumerId, char* key){ static void pushVgDataToHash(SArray *vgs, SHashObj *pHash, int64_t consumerId, char *key) {
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(vgs); SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(vgs);
SMqRebOutputVg outputVg = {consumerId, -1, pVgEp}; SMqRebOutputVg outputVg = {consumerId, -1, pVgEp};
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("[rebalance] sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64, mInfo("[rebalance] sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64, key, pVgEp->vgId, consumerId);
key, pVgEp->vgId, consumerId);
} }
static void processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { static void processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) {
int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
int32_t actualRemoved = 0; int32_t actualRemoved = 0;
for (int32_t i = 0; i < numOfRemoved; i++) { for (int32_t i = 0; i < numOfRemoved; i++) {
uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
if (pConsumerEp == NULL) { if (pConsumerEp == NULL) {
continue; continue;
@ -253,14 +252,15 @@ static void processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, c
} }
if (numOfRemoved != actualRemoved) { if (numOfRemoved != actualRemoved) {
mError("[rebalance] sub:%s mq rebalance removedNum:%d not matched with actual:%d", pOutput->pSub->key, numOfRemoved, actualRemoved); mError("[rebalance] sub:%s mq rebalance removedNum:%d not matched with actual:%d", pOutput->pSub->key, numOfRemoved,
actualRemoved);
} else { } else {
mInfo("[rebalance] sub:%s removed %d consumers", pOutput->pSub->key, numOfRemoved); mInfo("[rebalance] sub:%s removed %d consumers", pOutput->pSub->key, numOfRemoved);
} }
} }
static void processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) { static void processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) {
int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers); int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers);
for (int32_t i = 0; i < numOfNewConsumers; i++) { for (int32_t i = 0; i < numOfNewConsumers; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
@ -276,26 +276,26 @@ static void processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *
} }
static void processUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { static void processUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) {
int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs); int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs);
for (int32_t i = 0; i < numOfVgroups; i++) { for (int32_t i = 0; i < numOfVgroups; i++) {
pushVgDataToHash(pOutput->pSub->unassignedVgs, pHash, -1, pOutput->pSub->key); pushVgDataToHash(pOutput->pSub->unassignedVgs, pHash, -1, pOutput->pSub->key);
} }
} }
//static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ // static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){
// for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ // for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){
// SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); // SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i);
// SMqRebOutputVg outputVg = { // SMqRebOutputVg outputVg = {
// .oldConsumerId = pConsumerEp->consumerId, // .oldConsumerId = pConsumerEp->consumerId,
// .newConsumerId = pConsumerEp->consumerId, // .newConsumerId = pConsumerEp->consumerId,
// .pVgEp = pVgEp, // .pVgEp = pVgEp,
// }; // };
// taosArrayPush(pOutput->rebVgs, &outputVg); // taosArrayPush(pOutput->rebVgs, &outputVg);
// } // }
//} // }
static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt,
int32_t remainderVgCnt) { int32_t remainderVgCnt) {
int32_t cnt = 0; int32_t cnt = 0;
void *pIter = NULL; void *pIter = NULL;
@ -311,7 +311,7 @@ static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash,
taosArrayPush(pOutput->modifyConsumers, &pConsumerEp->consumerId); taosArrayPush(pOutput->modifyConsumers, &pConsumerEp->consumerId);
if (consumerVgNum > minVgCnt) { if (consumerVgNum > minVgCnt) {
if (cnt < remainderVgCnt) { if (cnt < remainderVgCnt) {
while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) { // pop until equal minVg + 1 while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) { // pop until equal minVg + 1
pushVgDataToHash(pConsumerEp->vgs, pHash, pConsumerEp->consumerId, pOutput->pSub->key); pushVgDataToHash(pConsumerEp->vgs, pHash, pConsumerEp->consumerId, pOutput->pSub->key);
} }
cnt++; cnt++;
@ -321,17 +321,17 @@ static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash,
} }
} }
} }
// putNoTransferToOutput(pOutput, pConsumerEp); // putNoTransferToOutput(pOutput, pConsumerEp);
} }
} }
static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput) {
int32_t totalVgNum = 0; int32_t totalVgNum = 0;
SVgObj* pVgroup = NULL; SVgObj *pVgroup = NULL;
void* pIter = NULL; void *pIter = NULL;
SArray* newVgs = taosArrayInit(0, POINTER_BYTES); SArray *newVgs = taosArrayInit(0, POINTER_BYTES);
while (1) { while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) { if (pIter == NULL) {
break; break;
} }
@ -342,7 +342,7 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){
} }
totalVgNum++; totalVgNum++;
SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
pVgEp->vgId = pVgroup->vgId; pVgEp->vgId = pVgroup->vgId;
taosArrayPush(newVgs, &pVgEp); taosArrayPush(newVgs, &pVgEp);
@ -358,17 +358,17 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){
int32_t j = 0; int32_t j = 0;
while (j < taosArrayGetSize(pConsumerEp->vgs)) { while (j < taosArrayGetSize(pConsumerEp->vgs)) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
bool find = false; bool find = false;
for(int32_t k = 0; k < taosArrayGetSize(newVgs); k++){ for (int32_t k = 0; k < taosArrayGetSize(newVgs); k++) {
SMqVgEp *pnewVgEp = taosArrayGetP(newVgs, k); SMqVgEp *pnewVgEp = taosArrayGetP(newVgs, k);
if(pVgEp->vgId == pnewVgEp->vgId){ if (pVgEp->vgId == pnewVgEp->vgId) {
tDeleteSMqVgEp(pnewVgEp); tDeleteSMqVgEp(pnewVgEp);
taosArrayRemove(newVgs, k); taosArrayRemove(newVgs, k);
find = true; find = true;
break; break;
} }
} }
if(!find){ if (!find) {
mInfo("[rebalance] processRemoveAddVgs old vgId:%d", pVgEp->vgId); mInfo("[rebalance] processRemoveAddVgs old vgId:%d", pVgEp->vgId);
tDeleteSMqVgEp(pVgEp); tDeleteSMqVgEp(pVgEp);
taosArrayRemove(pConsumerEp->vgs, j); taosArrayRemove(pConsumerEp->vgs, j);
@ -378,17 +378,17 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){
} }
} }
if(taosArrayGetSize(pOutput->pSub->unassignedVgs) == 0 && taosArrayGetSize(newVgs) != 0){ if (taosArrayGetSize(pOutput->pSub->unassignedVgs) == 0 && taosArrayGetSize(newVgs) != 0) {
taosArrayAddAll(pOutput->pSub->unassignedVgs, newVgs); taosArrayAddAll(pOutput->pSub->unassignedVgs, newVgs);
mInfo("[rebalance] processRemoveAddVgs add new vg num:%d", (int)taosArrayGetSize(newVgs)); mInfo("[rebalance] processRemoveAddVgs add new vg num:%d", (int)taosArrayGetSize(newVgs));
taosArrayDestroy(newVgs); taosArrayDestroy(newVgs);
}else{ } else {
taosArrayDestroyP(newVgs, (FDelete)tDeleteSMqVgEp); taosArrayDestroyP(newVgs, (FDelete)tDeleteSMqVgEp);
} }
return totalVgNum; return totalVgNum;
} }
static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput){ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key); // put all offset rows SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key); // put all offset rows
if (pSub == NULL) { if (pSub == NULL) {
return; return;
@ -406,16 +406,17 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S
for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) { for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) {
OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j); OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j);
bool jump = false; bool jump = false;
for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++){ for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEpNew->vgs, i); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEpNew->vgs, i);
if(pVgEp->vgId == d1->vgId){ if (pVgEp->vgId == d1->vgId) {
jump = true; jump = true;
mInfo("pSub->offsetRows jump, because consumer id:0x%"PRIx64 " and vgId:%d not change", pConsumerEp->consumerId, pVgEp->vgId); mInfo("pSub->offsetRows jump, because consumer id:0x%" PRIx64 " and vgId:%d not change",
pConsumerEp->consumerId, pVgEp->vgId);
break; break;
} }
} }
if(jump) continue; if (jump) continue;
bool find = false; bool find = false;
for (int i = 0; i < taosArrayGetSize(pOutput->pSub->offsetRows); i++) { for (int i = 0; i < taosArrayGetSize(pOutput->pSub->offsetRows); i++) {
OffsetRows *d2 = taosArrayGet(pOutput->pSub->offsetRows, i); OffsetRows *d2 = taosArrayGet(pOutput->pSub->offsetRows, i);
@ -424,11 +425,11 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S
d2->offset = d1->offset; d2->offset = d1->offset;
d2->ever = d1->ever; d2->ever = d1->ever;
find = true; find = true;
mInfo("pSub->offsetRows add vgId:%d, after:%"PRId64", before:%"PRId64, d2->vgId, d2->rows, d1->rows); mInfo("pSub->offsetRows add vgId:%d, after:%" PRId64 ", before:%" PRId64, d2->vgId, d2->rows, d1->rows);
break; break;
} }
} }
if(!find){ if (!find) {
taosArrayPush(pOutput->pSub->offsetRows, d1); taosArrayPush(pOutput->pSub->offsetRows, d1);
} }
} }
@ -437,7 +438,7 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S
mndReleaseSubscribe(pMnode, pSub); mndReleaseSubscribe(pMnode, pSub);
} }
static void printRebalanceLog(SMqRebOutputObj *pOutput){ static void printRebalanceLog(SMqRebOutputObj *pOutput) {
mInfo("sub:%s mq rebalance calculation completed, re-balanced vg", pOutput->pSub->key); mInfo("sub:%s mq rebalance calculation completed, re-balanced vg", pOutput->pSub->key);
for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
@ -451,7 +452,8 @@ static void printRebalanceLog(SMqRebOutputObj *pOutput){
if (pIter == NULL) break; if (pIter == NULL) break;
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
int32_t sz = taosArrayGetSize(pConsumerEp->vgs); int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
mInfo("sub:%s mq rebalance final cfg: consumer:0x%" PRIx64 " has %d vg", pOutput->pSub->key, pConsumerEp->consumerId, sz); mInfo("sub:%s mq rebalance final cfg: consumer:0x%" PRIx64 " has %d vg", pOutput->pSub->key,
pConsumerEp->consumerId, sz);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
mInfo("sub:%s mq rebalance final cfg: vg %d to consumer:0x%" PRIx64, pOutput->pSub->key, pVgEp->vgId, mInfo("sub:%s mq rebalance final cfg: vg %d to consumer:0x%" PRIx64, pOutput->pSub->key, pVgEp->vgId,
@ -460,8 +462,8 @@ static void printRebalanceLog(SMqRebOutputObj *pOutput){
} }
} }
static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, const char *pSubKey, static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, const char *pSubKey, int32_t *minVgCnt,
int32_t *minVgCnt, int32_t *remainderVgCnt){ int32_t *remainderVgCnt) {
int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
int32_t numOfAdded = taosArrayGetSize(pInput->pRebInfo->newConsumers); int32_t numOfAdded = taosArrayGetSize(pInput->pRebInfo->newConsumers);
int32_t numOfFinal = pInput->oldConsumerNum + numOfAdded - numOfRemoved; int32_t numOfFinal = pInput->oldConsumerNum + numOfAdded - numOfRemoved;
@ -473,11 +475,12 @@ static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, con
} else { } else {
mInfo("[rebalance] sub:%s no consumer subscribe this topic", pSubKey); mInfo("[rebalance] sub:%s no consumer subscribe this topic", pSubKey);
} }
mInfo("[rebalance] sub:%s mq rebalance %d vgroups, existed consumers:%d, added:%d, removed:%d, minVg:%d remainderVg:%d", mInfo(
pSubKey, totalVgNum, pInput->oldConsumerNum, numOfAdded, numOfRemoved, *minVgCnt, *remainderVgCnt); "[rebalance] sub:%s mq rebalance %d vgroups, existed consumers:%d, added:%d, removed:%d, minVg:%d remainderVg:%d",
pSubKey, totalVgNum, pInput->oldConsumerNum, numOfAdded, numOfRemoved, *minVgCnt, *remainderVgCnt);
} }
static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt){ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt) {
SMqRebOutputVg *pRebVg = NULL; SMqRebOutputVg *pRebVg = NULL;
void *pAssignIter = NULL; void *pAssignIter = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -498,8 +501,8 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min
pRebVg = (SMqRebOutputVg *)pAssignIter; pRebVg = (SMqRebOutputVg *)pAssignIter;
pRebVg->newConsumerId = pConsumerEp->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average", mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average", pRebVg->pVgEp->vgId,
pRebVg->pVgEp->vgId, pConsumerEp->consumerId); pConsumerEp->consumerId);
} }
} }
@ -519,13 +522,13 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min
pRebVg = (SMqRebOutputVg *)pAssignIter; pRebVg = (SMqRebOutputVg *)pAssignIter;
pRebVg->newConsumerId = pConsumerEp->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average + 1", mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average + 1", pRebVg->pVgEp->vgId,
pRebVg->pVgEp->vgId, pConsumerEp->consumerId); pConsumerEp->consumerId);
} }
} }
taosHashCancelIterate(pOutput->pSub->consumerHash, pIter); taosHashCancelIterate(pOutput->pSub->consumerHash, pIter);
if(pAssignIter != NULL){ if (pAssignIter != NULL) {
mError("[rebalance]sub:%s assign iter is not NULL, never should reach here", pOutput->pSub->key); mError("[rebalance]sub:%s assign iter is not NULL, never should reach here", pOutput->pSub->key);
} }
while (1) { while (1) {
@ -534,19 +537,19 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min
break; break;
} }
SMqRebOutputVg* pRebOutput = (SMqRebOutputVg *)pAssignIter; SMqRebOutputVg *pRebOutput = (SMqRebOutputVg *)pAssignIter;
taosArrayPush(pOutput->rebVgs, pRebOutput); taosArrayPush(pOutput->rebVgs, pRebOutput);
if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed if (taosHashGetSize(pOutput->pSub->consumerHash) == 0) { // if all consumer is removed
taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); // put all vg into unassigned taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); // put all vg into unassigned
} }
} }
} }
static void mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { static void mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
int32_t totalVgNum = processRemoveAddVgs(pMnode, pOutput); int32_t totalVgNum = processRemoveAddVgs(pMnode, pOutput);
const char *pSubKey = pOutput->pSub->key; const char *pSubKey = pOutput->pSub->key;
int32_t minVgCnt = 0; int32_t minVgCnt = 0;
int32_t remainderVgCnt = 0; int32_t remainderVgCnt = 0;
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
@ -561,14 +564,14 @@ static void mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebO
taosHashCleanup(pHash); taosHashCleanup(pHash);
} }
static int32_t presistConsumerByType(STrans *pTrans, SArray* consumers, int8_t type, char *cgroup, char *topic){ static int32_t presistConsumerByType(STrans *pTrans, SArray *consumers, int8_t type, char *cgroup, char *topic) {
int32_t code = 0; int32_t code = 0;
SMqConsumerObj *pConsumerNew = NULL; SMqConsumerObj *pConsumerNew = NULL;
int32_t consumerNum = taosArrayGetSize(consumers); int32_t consumerNum = taosArrayGetSize(consumers);
for (int32_t i = 0; i < consumerNum; i++) { for (int32_t i = 0; i < consumerNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(consumers, i); int64_t consumerId = *(int64_t *)taosArrayGet(consumers, i);
pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, type, topic, NULL); pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, type, topic, NULL);
if (pConsumerNew == NULL){ if (pConsumerNew == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto END; goto END;
} }
@ -587,7 +590,7 @@ END:
return code; return code;
} }
static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput, char *cgroup, char *topic){ static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput, char *cgroup, char *topic) {
int32_t code = presistConsumerByType(pTrans, pOutput->modifyConsumers, CONSUMER_UPDATE_REB, cgroup, NULL); int32_t code = presistConsumerByType(pTrans, pOutput->modifyConsumers, CONSUMER_UPDATE_REB, cgroup, NULL);
if (code != 0) { if (code != 0) {
return code; return code;
@ -603,10 +606,10 @@ static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput
static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) {
struct SSubplan *pPlan = NULL; struct SSubplan *pPlan = NULL;
int32_t code = 0; int32_t code = 0;
STrans *pTrans = NULL; STrans *pTrans = NULL;
if(strcmp(pOutput->pSub->qmsg, "") != 0){ if (strcmp(pOutput->pSub->qmsg, "") != 0) {
code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan); code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan);
if (code != 0) { if (code != 0) {
terrno = code; terrno = code;
@ -660,7 +663,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
code = mndTransPrepare(pMnode, pTrans); code = mndTransPrepare(pMnode, pTrans);
END: END:
nodesDestroyNode((SNode*)pPlan); nodesDestroyNode((SNode *)pPlan);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; return code;
} }
@ -672,24 +675,24 @@ static void freeRebalanceItem(void *param) {
} }
// type = 0 remove type = 1 add // type = 0 remove type = 1 add
static void buildRebInfo(SHashObj* rebSubHash, SArray* topicList, int8_t type, char *group, int64_t consumerId){ static void buildRebInfo(SHashObj *rebSubHash, SArray *topicList, int8_t type, char *group, int64_t consumerId) {
int32_t topicNum = taosArrayGetSize(topicList); int32_t topicNum = taosArrayGetSize(topicList);
for (int32_t i = 0; i < topicNum; i++) { for (int32_t i = 0; i < topicNum; i++) {
char key[TSDB_SUBSCRIBE_KEY_LEN]; char key[TSDB_SUBSCRIBE_KEY_LEN];
char *removedTopic = taosArrayGetP(topicList, i); char *removedTopic = taosArrayGetP(topicList, i);
mndMakeSubscribeKey(key, group, removedTopic); mndMakeSubscribeKey(key, group, removedTopic);
SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key); SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key);
if(type == 0) if (type == 0)
taosArrayPush(pRebSub->removedConsumers, &consumerId); taosArrayPush(pRebSub->removedConsumers, &consumerId);
else if(type == 1) else if (type == 1)
taosArrayPush(pRebSub->newConsumers, &consumerId); taosArrayPush(pRebSub->newConsumers, &consumerId);
} }
} }
static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHashObj* rebSubHash){ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHashObj *rebSubHash) {
int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics); int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics);
for (int32_t i = 0; i < newTopicNum; i++) { for (int32_t i = 0; i < newTopicNum; i++) {
char * topic = taosArrayGetP(pConsumer->currentTopics, i); char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
if (pSub == NULL) { if (pSub == NULL) {
continue; continue;
@ -702,7 +705,7 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash
for (int32_t j = 0; j < vgNum; j++) { for (int32_t j = 0; j < vgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
SVgObj * pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId);
if (!pVgroup) { if (!pVgroup) {
char key[TSDB_SUBSCRIBE_KEY_LEN]; char key[TSDB_SUBSCRIBE_KEY_LEN];
mndMakeSubscribeKey(key, pConsumer->cgroup, topic); mndMakeSubscribeKey(key, pConsumer->cgroup, topic);
@ -716,7 +719,7 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash
} }
} }
static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *rebSubHash) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SMqConsumerObj *pConsumer = NULL; SMqConsumerObj *pConsumer = NULL;
@ -732,17 +735,19 @@ static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) {
int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1);
int32_t status = atomic_load_32(&pConsumer->status); int32_t status = atomic_load_32(&pConsumer->status);
mDebug("[rebalance] check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", mDebug("[rebalance] check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64
pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, hbStatus); ", hbstatus:%d",
pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime,
pConsumer->createTime, hbStatus);
if (status == MQ_CONSUMER_STATUS_READY) { if (status == MQ_CONSUMER_STATUS_READY) {
if (taosArrayGetSize(pConsumer->assignedTopics) == 0) {// unsubscribe or close if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close
mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, &pMsg->info); mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, &pMsg->info);
} else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { } else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) {
taosRLockLatch(&pConsumer->lock); taosRLockLatch(&pConsumer->lock);
buildRebInfo(rebSubHash, pConsumer->currentTopics, 0, pConsumer->cgroup, pConsumer->consumerId); buildRebInfo(rebSubHash, pConsumer->currentTopics, 0, pConsumer->cgroup, pConsumer->consumerId);
taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pConsumer->lock);
}else{ } else {
checkForVgroupSplit(pMnode, pConsumer, rebSubHash); checkForVgroupSplit(pMnode, pConsumer, rebSubHash);
} }
} else if (status == MQ_CONSUMER_STATUS_LOST) { } else if (status == MQ_CONSUMER_STATUS_LOST) {
@ -762,8 +767,7 @@ static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) {
bool mndRebTryStart() { bool mndRebTryStart() {
int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1);
if (old > 0) mInfo("[rebalance] counter old val:%d", old) if (old > 0) mInfo("[rebalance] counter old val:%d", old) return old == 0;
return old == 0;
} }
void mndRebCntInc() { void mndRebCntInc() {
@ -776,7 +780,7 @@ void mndRebCntDec() {
if (val > 0) mInfo("[rebalance] cnt sub, value:%d", val) if (val > 0) mInfo("[rebalance] cnt sub, value:%d", val)
} }
static void clearRebOutput(SMqRebOutputObj *rebOutput){ static void clearRebOutput(SMqRebOutputObj *rebOutput) {
taosArrayDestroy(rebOutput->newConsumers); taosArrayDestroy(rebOutput->newConsumers);
taosArrayDestroy(rebOutput->modifyConsumers); taosArrayDestroy(rebOutput->modifyConsumers);
taosArrayDestroy(rebOutput->removedConsumers); taosArrayDestroy(rebOutput->removedConsumers);
@ -800,15 +804,15 @@ static int32_t initRebOutput(SMqRebOutputObj *rebOutput) {
} }
// This function only works when there are dirty consumers // This function only works when there are dirty consumers
static void checkConsumer(SMnode *pMnode, SMqSubscribeObj* pSub){ static void checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) {
void *pIter = NULL; void *pIter = NULL;
while (1) { while (1) {
pIter = taosHashIterate(pSub->consumerHash, pIter); pIter = taosHashIterate(pSub->consumerHash, pIter);
if (pIter == NULL) { if (pIter == NULL) {
break; break;
} }
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pConsumerEp->consumerId); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pConsumerEp->consumerId);
if (pConsumer != NULL) { if (pConsumer != NULL) {
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
@ -822,8 +826,8 @@ static void checkConsumer(SMnode *pMnode, SMqSubscribeObj* pSub){
} }
} }
static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput){ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) {
const char *key = rebInput->pRebInfo->key; const char *key = rebInput->pRebInfo->key;
SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key); SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key);
if (pSub == NULL) { if (pSub == NULL) {
@ -869,14 +873,13 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu
} }
static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
int code = 0; int code = 0;
void *pIter = NULL; void *pIter = NULL;
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
mDebug("[rebalance] start to process mq timer") mDebug("[rebalance] start to process mq timer")
if (!mndRebTryStart()) { if (!mndRebTryStart()) {
mInfo("[rebalance] mq rebalance already in progress, do nothing") mInfo("[rebalance] mq rebalance already in progress, do nothing") return code;
return code;
} }
SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
@ -900,13 +903,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
SMqRebInputObj rebInput = {0}; SMqRebInputObj rebInput = {0};
SMqRebOutputObj rebOutput = {0}; SMqRebOutputObj rebOutput = {0};
code = initRebOutput(&rebOutput); code = initRebOutput(&rebOutput);
if (code != 0){ if (code != 0) {
goto END; goto END;
} }
rebInput.pRebInfo = (SMqRebInfo*)pIter; rebInput.pRebInfo = (SMqRebInfo *)pIter;
if (buildRebOutput(pMnode, &rebInput, &rebOutput) != 0){ if (buildRebOutput(pMnode, &rebInput, &rebOutput) != 0) {
continue; continue;
} }
@ -931,12 +934,12 @@ END:
return code; return code;
} }
static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans){ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) {
void* pIter = NULL; void *pIter = NULL;
SVgObj* pVgObj = NULL; SVgObj *pVgObj = NULL;
int32_t ret = 0; int32_t ret = 0;
while (1) { while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgObj); pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgObj);
if (pIter == NULL) { if (pIter == NULL) {
break; break;
} }
@ -946,7 +949,7 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran
continue; continue;
} }
SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq)); SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq));
if(pReq == NULL){ if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
ret = -1; ret = -1;
goto END; goto END;
@ -957,7 +960,8 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran
memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
STransAction action = {0}; STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);; action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
;
action.pCont = pReq; action.pCont = pReq;
action.contLen = sizeof(SMqVDeleteReq); action.contLen = sizeof(SMqVDeleteReq);
action.msgType = TDMT_VND_TMQ_DELETE_SUB; action.msgType = TDMT_VND_TMQ_DELETE_SUB;
@ -969,13 +973,13 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran
goto END; goto END;
} }
} }
END: END:
sdbRelease(pMnode->pSdb, pVgObj); sdbRelease(pMnode->pSdb, pVgObj);
sdbCancelFetch(pMnode->pSdb, pIter); sdbCancelFetch(pMnode->pSdb, pIter);
return ret; return ret;
} }
static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic){ static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) {
void *pIter = NULL; void *pIter = NULL;
SMqConsumerObj *pConsumer = NULL; SMqConsumerObj *pConsumer = NULL;
int ret = 0; int ret = 0;
@ -1012,10 +1016,10 @@ END:
} }
static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
SMDropCgroupReq dropReq = {0}; SMDropCgroupReq dropReq = {0};
STrans *pTrans = NULL; STrans *pTrans = NULL;
int32_t code = TSDB_CODE_ACTION_IN_PROGRESS; int32_t code = TSDB_CODE_ACTION_IN_PROGRESS;
if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
@ -1262,7 +1266,6 @@ int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName) {
pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub); pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub);
if (pIter == NULL) break; if (pIter == NULL) break;
char topic[TSDB_TOPIC_FNAME_LEN] = {0}; char topic[TSDB_TOPIC_FNAME_LEN] = {0};
char cgroup[TSDB_CGROUP_LEN] = {0}; char cgroup[TSDB_CGROUP_LEN] = {0};
mndSplitSubscribeKey(pSub->key, topic, cgroup, true); mndSplitSubscribeKey(pSub->key, topic, cgroup, true);
@ -1340,7 +1343,8 @@ END:
return code; return code;
} }
static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t consumerId, const char* topic, const char* cgroup, SArray* vgs, SArray *offsetRows){ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic,
const char *cgroup, SArray *vgs, SArray *offsetRows) {
int32_t sz = taosArrayGetSize(vgs); int32_t sz = taosArrayGetSize(vgs);
for (int32_t j = 0; j < sz; j++) { for (int32_t j = 0; j < sz; j++) {
SMqVgEp *pVgEp = taosArrayGetP(vgs, j); SMqVgEp *pVgEp = taosArrayGetP(vgs, j);
@ -1359,37 +1363,37 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t cons
colDataSetVal(pColInfo, *numOfRows, (const char *)&pVgEp->vgId, false); colDataSetVal(pColInfo, *numOfRows, (const char *)&pVgEp->vgId, false);
// consumer id // consumer id
char consumerIdHex[32] = {0}; char consumerIdHex[32] = {0};
sprintf(varDataVal(consumerIdHex), "0x%"PRIx64, consumerId); sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, consumerId);
varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)consumerIdHex, consumerId == -1); colDataSetVal(pColInfo, *numOfRows, (const char *)consumerIdHex, consumerId == -1);
mInfo("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), mInfo("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), consumerId,
consumerId, varDataVal(cgroup), pVgEp->vgId); varDataVal(cgroup), pVgEp->vgId);
// offset // offset
OffsetRows *data = NULL; OffsetRows *data = NULL;
for(int i = 0; i < taosArrayGetSize(offsetRows); i++){ for (int i = 0; i < taosArrayGetSize(offsetRows); i++) {
OffsetRows *tmp = taosArrayGet(offsetRows, i); OffsetRows *tmp = taosArrayGet(offsetRows, i);
if(tmp->vgId != pVgEp->vgId){ if (tmp->vgId != pVgEp->vgId) {
mError("mnd show subscriptions: do not find vgId:%d, %d in offsetRows", tmp->vgId, pVgEp->vgId); mError("mnd show subscriptions: do not find vgId:%d, %d in offsetRows", tmp->vgId, pVgEp->vgId);
continue; continue;
} }
data = tmp; data = tmp;
} }
if(data){ if (data) {
// vg id // vg id
char buf[TSDB_OFFSET_LEN*2 + VARSTR_HEADER_SIZE] = {0}; char buf[TSDB_OFFSET_LEN * 2 + VARSTR_HEADER_SIZE] = {0};
tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset); tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset);
sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%"PRId64, data->ever); sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%" PRId64, data->ever);
varDataSetLen(buf, strlen(varDataVal(buf))); varDataSetLen(buf, strlen(varDataVal(buf)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)buf, false); colDataSetVal(pColInfo, *numOfRows, (const char *)buf, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)&data->rows, false); colDataSetVal(pColInfo, *numOfRows, (const char *)&data->rows, false);
}else{ } else {
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetNULL(pColInfo, *numOfRows); colDataSetNULL(pColInfo, *numOfRows);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@ -1435,7 +1439,8 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock
if (pIter == NULL) break; if (pIter == NULL) break;
pConsumerEp = (SMqConsumerEp *)pIter; pConsumerEp = (SMqConsumerEp *)pIter;
buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs, pConsumerEp->offsetRows); buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs,
pConsumerEp->offsetRows);
} }
buildResult(pBlock, &numOfRows, -1, topic, cgroup, pSub->unassignedVgs, pSub->offsetRows); buildResult(pBlock, &numOfRows, -1, topic, cgroup, pSub->unassignedVgs, pSub->offsetRows);

12
source/dnode/snode/src/snode.c
View File

@ -41,7 +41,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
SCheckpointInfo *pChkInfo = &pTask->chkInfo; SCheckpointInfo *pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask); tqSetRestoreVersionInfo(pTask);
char* p = streamTaskGetStatus(pTask)->name; char *p = streamTaskGetStatus(pTask)->name;
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " nextProcessVer:%" PRId64
@ -71,7 +71,8 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
startRsync(); startRsync();
pSnode->msgCb = pOption->msgCb; pSnode->msgCb = pOption->msgCb;
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndExpandTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndExpandTask, tqExpandStreamTask, SNODE_HANDLE,
taosGetTimestampMs(), tqStartTaskCompleteCallback);
if (pSnode->pMeta == NULL) { if (pSnode->pMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL; goto FAIL;
@ -138,9 +139,10 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) {
int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) {
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_STREAM_TASK_DEPLOY: { case TDMT_STREAM_TASK_DEPLOY: {
void * pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t len = pMsg->contLen - sizeof(SMsgHead); int32_t len = pMsg->contLen - sizeof(SMsgHead);
return tqStreamTaskProcessDeployReq(pSnode->pMeta, &pSnode->msgCb,pMsg->info.conn.applyIndex, pReq, len, true, true); return tqStreamTaskProcessDeployReq(pSnode->pMeta, &pSnode->msgCb, pMsg->info.conn.applyIndex, pReq, len, true,
true);
} }
case TDMT_STREAM_TASK_DROP: case TDMT_STREAM_TASK_DROP:
@ -159,4 +161,4 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) {
ASSERT(0); ASSERT(0);
} }
return 0; return 0;
} }

10
source/dnode/vnode/src/meta/metaCache.c
View File

@ -686,8 +686,8 @@ int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int
} }
// add to cache. // add to cache.
taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL, (void)taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL,
TAOS_LRU_PRIORITY_LOW, NULL); TAOS_LRU_PRIORITY_LOW, NULL);
_end: _end:
taosThreadMutexUnlock(pLock); taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid, metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid,
@ -909,7 +909,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) { bool metaTbInFilterCache(SMeta* pMeta, const void* key, int8_t type) {
if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) { if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) {
return true; return true;
} }
@ -921,7 +921,7 @@ bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) {
return false; return false;
} }
int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) { int32_t metaPutTbToFilterCache(SMeta* pMeta, const void* key, int8_t type) {
if (type == 0) { if (type == 0) {
return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0); return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0);
} }
@ -933,7 +933,7 @@ int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) {
return 0; return 0;
} }
int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type) { int32_t metaSizeOfTbFilterCache(SMeta* pMeta, int8_t type) {
if (type == 0) { if (type == 0) {
return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb);
} }

88
source/dnode/vnode/src/meta/metaTable.c
View File

@ -647,13 +647,18 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SCtbIdxKey *table = (SCtbIdxKey *)pKey; SCtbIdxKey *table = (SCtbIdxKey *)pKey;
STagVal tagVal = {.cid = pCol->colId}; STagVal tagVal = {.cid = pCol->colId};
tTagGet((const STag *)pVal, &tagVal); if (tTagGet((const STag *)pVal, &tagVal)) {
if (IS_VAR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
pTagData = tagVal.pData; pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData; nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pCol->type].bytes;
}
} else { } else {
pTagData = &(tagVal.i64); if (!IS_VAR_DATA_TYPE(pCol->type)) {
nTagData = tDataTypes[pCol->type].bytes; nTagData = tDataTypes[pCol->type].bytes;
}
} }
rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey);
tdbFree(pKey); tdbFree(pKey);
@ -779,13 +784,18 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
SCtbIdxKey *table = (SCtbIdxKey *)pKey; SCtbIdxKey *table = (SCtbIdxKey *)pKey;
STagVal tagVal = {.cid = pCol->colId}; STagVal tagVal = {.cid = pCol->colId};
tTagGet((const STag *)pVal, &tagVal); if (tTagGet((const STag *)pVal, &tagVal)) {
if (IS_VAR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
pTagData = tagVal.pData; pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData; nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pCol->type].bytes;
}
} else { } else {
pTagData = &(tagVal.i64); if (!IS_VAR_DATA_TYPE(pCol->type)) {
nTagData = tDataTypes[pCol->type].bytes; nTagData = tDataTypes[pCol->type].bytes;
}
} }
rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey);
tdbFree(pKey); tdbFree(pKey);
@ -1272,13 +1282,18 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p
int32_t nTagData = 0; int32_t nTagData = 0;
STagVal tagVal = {.cid = pTagColumn->colId}; STagVal tagVal = {.cid = pTagColumn->colId};
tTagGet((const STag *)e.ctbEntry.pTags, &tagVal); if (tTagGet((const STag *)e.ctbEntry.pTags, &tagVal)) {
if (IS_VAR_DATA_TYPE(pTagColumn->type)) { if (IS_VAR_DATA_TYPE(pTagColumn->type)) {
pTagData = tagVal.pData; pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData; nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pTagColumn->type].bytes;
}
} else { } else {
pTagData = &(tagVal.i64); if (!IS_VAR_DATA_TYPE(pTagColumn->type)) {
nTagData = tDataTypes[pTagColumn->type].bytes; nTagData = tDataTypes[pTagColumn->type].bytes;
}
} }
if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid,
@ -2036,13 +2051,18 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb
int32_t nTagData = 0; int32_t nTagData = 0;
STagVal tagVal = {.cid = pCol->colId}; STagVal tagVal = {.cid = pCol->colId};
tTagGet((const STag *)pVal, &tagVal); if (tTagGet((const STag *)pVal, &tagVal)) {
if (IS_VAR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
pTagData = tagVal.pData; pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData; nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pCol->type].bytes;
}
} else { } else {
pTagData = &(tagVal.i64); if (!IS_VAR_DATA_TYPE(pCol->type)) {
nTagData = tDataTypes[pCol->type].bytes; nTagData = tDataTypes[pCol->type].bytes;
}
} }
if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) { if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) {
tdbFree(pKey); tdbFree(pKey);
@ -2497,19 +2517,25 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
goto end; goto end;
} else { } else {
for (int i = 0; i < pTagSchema->nCols; i++) { for (int i = 0; i < pTagSchema->nCols; i++) {
pTagData = NULL;
nTagData = 0;
pTagColumn = &pTagSchema->pSchema[i]; pTagColumn = &pTagSchema->pSchema[i];
if (!IS_IDX_ON(pTagColumn)) continue; if (!IS_IDX_ON(pTagColumn)) continue;
STagVal tagVal = {.cid = pTagColumn->colId}; STagVal tagVal = {.cid = pTagColumn->colId};
tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); if (tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal)) {
if (IS_VAR_DATA_TYPE(pTagColumn->type)) { if (IS_VAR_DATA_TYPE(pTagColumn->type)) {
pTagData = tagVal.pData; pTagData = tagVal.pData;
nTagData = (int32_t)tagVal.nData; nTagData = (int32_t)tagVal.nData;
} else {
pTagData = &(tagVal.i64);
nTagData = tDataTypes[pTagColumn->type].bytes;
}
} else { } else {
pTagData = &(tagVal.i64); if (!IS_VAR_DATA_TYPE(pTagColumn->type)) {
nTagData = tDataTypes[pTagColumn->type].bytes; nTagData = tDataTypes[pTagColumn->type].bytes;
}
} }
if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type,
pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) {
ret = -1; ret = -1;

5
source/dnode/vnode/src/tq/tq.c
View File

@ -697,6 +697,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
} else { } else {
tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
req.newConsumerId); req.newConsumerId);
atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId);
atomic_store_32(&pHandle->epoch, 0); atomic_store_32(&pHandle->epoch, 0);
tqUnregisterPushHandle(pTq, pHandle); tqUnregisterPushHandle(pTq, pHandle);
@ -963,8 +964,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s",
pTask->streamTaskId.taskId, pTask->id.idStr); pTask->streamTaskId.taskId, pTask->id.idStr);
tqDebug("s-task:%s fill-history task set status to be dropping and drop it", id); tqDebug("s-task:%s fill-history task set status to be dropping", id);
streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
atomic_store_32(&pTask->status.inScanHistorySentinel, 0); atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);

51
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -19,13 +19,13 @@
typedef struct SMStreamCheckpointReadyRspMsg { typedef struct SMStreamCheckpointReadyRspMsg {
SMsgHead head; SMsgHead head;
int64_t streamId; int64_t streamId;
int32_t upstreamTaskId; int32_t upstreamTaskId;
int32_t upstreamNodeId; int32_t upstreamNodeId;
int32_t downstreamTaskId; int32_t downstreamTaskId;
int32_t downstreamNodeId; int32_t downstreamNodeId;
int64_t checkpointId; int64_t checkpointId;
int32_t transId; int32_t transId;
} SMStreamCheckpointReadyRspMsg; } SMStreamCheckpointReadyRspMsg;
static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
@ -160,7 +160,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask == NULL || *ppTask == NULL) { if (ppTask == NULL || *ppTask == NULL) {
tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId, req.taskId); tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId,
req.taskId);
rsp.code = TSDB_CODE_SUCCESS; rsp.code = TSDB_CODE_SUCCESS;
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
@ -194,7 +195,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
void* pReqTask = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); void* pReqTask = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry));
if (pReqTask != NULL) { if (pReqTask != NULL) {
tqDebug("s-task:%s (vgId:%d) already update in transId:%d, discard the nodeEp update msg", idstr, vgId, req.transId); tqDebug("s-task:%s (vgId:%d) already update in transId:%d, discard the nodeEp update msg", idstr, vgId,
req.transId);
rsp.code = TSDB_CODE_SUCCESS; rsp.code = TSDB_CODE_SUCCESS;
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
@ -838,10 +840,11 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
streamTaskClearCheckInfo(pTask, true); streamTaskClearCheckInfo(pTask, true);
// clear flag set during do checkpoint, and open inputQ for all upstream tasks // clear flag set during do checkpoint, and open inputQ for all upstream tasks
SStreamTaskState *pState = streamTaskGetStatus(pTask); SStreamTaskState* pState = streamTaskGetStatus(pTask);
if (pState->state == TASK_STATUS__CK) { if (pState->state == TASK_STATUS__CK) {
int32_t tranId = 0; int32_t tranId = 0;
int64_t activeChkId = 0; int64_t activeChkId = 0;
@ -866,7 +869,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
} }
int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg->pCont; SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId);
if (pTask == NULL) { if (pTask == NULL) {
@ -883,7 +886,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready", tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready",
pTask->id.idStr, (int32_t)pReq->downstreamTaskId); pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_STREAM_TASK_IVLD_STATUS); streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -895,14 +899,15 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
int64_t checkpointId = 0; int64_t checkpointId = 0;
streamTaskGetActiveCheckpointInfo(pTask, &transId, &checkpointId); streamTaskGetActiveCheckpointInfo(pTask, &transId, &checkpointId);
ASSERT (checkpointId == pReq->checkpointId); ASSERT(checkpointId == pReq->checkpointId);
if (streamTaskAlreadySendTrigger(pTask, pReq->downstreamNodeId)) { if (streamTaskAlreadySendTrigger(pTask, pReq->downstreamNodeId)) {
// re-send the lost checkpoint-trigger msg to downstream task // re-send the lost checkpoint-trigger msg to downstream task
tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr, tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
(int32_t)pReq->downstreamTaskId, checkpointId, transId); (int32_t)pReq->downstreamTaskId, checkpointId, transId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_SUCCESS); streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
} else { // not send checkpoint-trigger yet, wait TSDB_CODE_SUCCESS);
} else { // not send checkpoint-trigger yet, wait
int32_t recv = 0, total = 0; int32_t recv = 0, total = 0;
streamTaskGetTriggerRecvStatus(pTask, &recv, &total); streamTaskGetTriggerRecvStatus(pTask, &recv, &total);
@ -914,7 +919,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"sending checkpoint-source/trigger", "sending checkpoint-source/trigger",
pTask->id.idStr, recv, total); pTask->id.idStr, recv, total);
} }
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
} }
} else { // upstream not recv the checkpoint-source/trigger till now } else { // upstream not recv the checkpoint-source/trigger till now
ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT); ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT);
@ -922,7 +928,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
"upstream sending checkpoint-source/trigger", "upstream sending checkpoint-source/trigger",
pTask->id.idStr); pTask->id.idStr);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -940,8 +947,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%"PRId64", transId:%d", tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d",
pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId); pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId);
streamTaskProcessCheckpointTriggerRsp(pTask, pRsp); streamTaskProcessCheckpointTriggerRsp(pTask, pRsp);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -998,7 +1005,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
int32_t level = pTask->info.taskLevel; int32_t level = pTask->info.taskLevel;
if (level == TASK_LEVEL__SINK) { if (level == TASK_LEVEL__SINK) {
ASSERT (status != TASK_STATUS__UNINIT); ASSERT(status != TASK_STATUS__UNINIT);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return 0; return 0;
} }
@ -1025,7 +1032,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
streamTrySchedExec(pTask); streamTrySchedExec(pTask);
} }
} else { } else {
ASSERT (status != TASK_STATUS__UNINIT); ASSERT(status != TASK_STATUS__UNINIT);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -1063,7 +1070,7 @@ int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return d
int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {return doProcessDummyRspMsg(pMeta, pMsg);} int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;

1
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -771,6 +771,7 @@ int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrap
STSchema *pTSchema = NULL; STSchema *pTSchema = NULL;
code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTSchema); code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTSchema);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosThreadMutexUnlock(&pTsdb->lruMutex);
terrno = code; terrno = code;
return -1; return -1;
} }

92
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -49,7 +49,8 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter);
static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,
STsdbReader* pReader); STsdbReader* pReader);
static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader);
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,
STsdbReader* pReader);
static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,
SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* id); SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* id);
static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList,
@ -68,7 +69,8 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRo
STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow); STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow);
static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey, static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey,
STsdbReader* pReader); STsdbReader* pReader);
static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,
STsdbReader* pReader);
static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost); static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost);
static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr, static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr,
@ -126,7 +128,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_
} }
// for test purpose, todo remove it // for test purpose, todo remove it
static int32_t tGetPrimaryKeyIndex(uint8_t *p, SPrimaryKeyIndex *index) { static int32_t tGetPrimaryKeyIndex(uint8_t* p, SPrimaryKeyIndex* index) {
int32_t n = 0; int32_t n = 0;
n += tGetI8(p + n, &index->type); n += tGetI8(p + n, &index->type);
n += tGetU32v(p + n, &index->offset); n += tGetU32v(p + n, &index->offset);
@ -137,7 +139,7 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
SPrimaryKeyIndex indices[TD_MAX_PK_COLS]; SPrimaryKeyIndex indices[TD_MAX_PK_COLS];
ASSERT(pKey->numOfPKs <= TD_MAX_PK_COLS); ASSERT(pKey->numOfPKs <= TD_MAX_PK_COLS);
uint8_t *data = pRow->data; uint8_t* data = pRow->data;
for (int32_t i = 0; i < pRow->numOfPKs; i++) { for (int32_t i = 0; i < pRow->numOfPKs; i++) {
data += tGetPrimaryKeyIndex(data, &indices[i]); data += tGetPrimaryKeyIndex(data, &indices[i]);
} }
@ -146,7 +148,7 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
for (int32_t i = 0; i < pRow->numOfPKs; i++) { for (int32_t i = 0; i < pRow->numOfPKs; i++) {
pKey->pks[i].type = indices[i].type; pKey->pks[i].type = indices[i].type;
uint8_t *tdata = data + indices[i].offset; uint8_t* tdata = data + indices[i].offset;
if (pRow->flag >> 4) { if (pRow->flag >> 4) {
tdata += tGetI16v(tdata, NULL); tdata += tGetI16v(tdata, NULL);
} }
@ -363,9 +365,7 @@ _err:
return code; return code;
} }
bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp) { bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) { return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type); }
return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type);
}
void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) { void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) {
pIter->order = order; pIter->order = order;
@ -1077,10 +1077,10 @@ static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInf
pLast->type = pSupp->pk.type; pLast->type = pSupp->pk.type;
if (IS_VAR_DATA_TYPE(pFirst->type)) { if (IS_VAR_DATA_TYPE(pFirst->type)) {
pFirst->pData = (uint8_t*) varDataVal(pBlockInfo->firstPk.pData); pFirst->pData = (uint8_t*)varDataVal(pBlockInfo->firstPk.pData);
pFirst->nData = varDataLen(pBlockInfo->firstPk.pData); pFirst->nData = varDataLen(pBlockInfo->firstPk.pData);
pLast->pData = (uint8_t*) varDataVal(pBlockInfo->lastPk.pData); pLast->pData = (uint8_t*)varDataVal(pBlockInfo->lastPk.pData);
pLast->nData = varDataLen(pBlockInfo->lastPk.pData); pLast->nData = varDataLen(pBlockInfo->lastPk.pData);
} else { } else {
pFirst->val = pBlockInfo->firstPk.val; pFirst->val = pBlockInfo->firstPk.val;
@ -1417,7 +1417,8 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
} }
// todo: this attribute could be acquired during extractin the global ordered block list. // todo: this attribute could be acquired during extractin the global ordered block list.
static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order, int32_t pkType, int32_t numOfPk) { static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order, int32_t pkType,
int32_t numOfPk) {
// it is the last block in current file, no chance to overlap with neighbor blocks. // it is the last block in current file, no chance to overlap with neighbor blocks.
if (ASCENDING_TRAVERSE(order)) { if (ASCENDING_TRAVERSE(order)) {
if (pBlock->lastKey == pRec->firstKey.key.ts) { if (pBlock->lastKey == pRec->firstKey.key.ts) {
@ -1429,7 +1430,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
v1.val = pBlock->lastPk.val; v1.val = pBlock->lastPk.val;
} }
return (tValueCompare(&v1, &pRec->firstKey.key.pks[0]) == 0); return (tValueCompare(&v1, &pRec->firstKey.key.pks[0]) == 0);
} else { // no pk } else { // no pk
return true; return true;
} }
} else { } else {
@ -1445,7 +1446,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
v1.val = pBlock->firstPk.val; v1.val = pBlock->firstPk.val;
} }
return (tValueCompare(&v1, &pRec->lastKey.key.pks[0]) == 0); return (tValueCompare(&v1, &pRec->lastKey.key.pks[0]) == 0);
} else { // no pk } else { // no pk
return true; return true;
} }
} else { } else {
@ -1585,7 +1586,7 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
*copied = false; *copied = false;
bool asc = (pReader->info.order == TSDB_ORDER_ASC); bool asc = (pReader->info.order == TSDB_ORDER_ASC);
if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) { if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) {
int32_t step = ASCENDING_TRAVERSE(pReader->info.order)? 1 : -1; int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1;
SRowKey nextRowKey; SRowKey nextRowKey;
tColRowGetKey(pBlockData, pDumpInfo->rowIndex + step, &nextRowKey); tColRowGetKey(pBlockData, pDumpInfo->rowIndex + step, &nextRowKey);
@ -2077,7 +2078,8 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan
return code; return code;
} }
static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo,
STsdbReader* pReader) {
SRowKey rowKey = {0}; SRowKey rowKey = {0};
while (1) { while (1) {
@ -2119,7 +2121,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
startKey.key = pBlockScanInfo->lastProcKey; startKey.key = pBlockScanInfo->lastProcKey;
startKey.version = asc ? pReader->info.verRange.minVer : pReader->info.verRange.maxVer; startKey.version = asc ? pReader->info.verRange.minVer : pReader->info.verRange.maxVer;
if ((asc && (startKey.key.ts < pWindow->skey)) || ((!asc) && startKey.key.ts > pWindow->ekey)) { if ((asc && (startKey.key.ts < pWindow->skey)) || ((!asc) && startKey.key.ts > pWindow->ekey)) {
startKey.key.ts = asc? pWindow->skey:pWindow->ekey; startKey.key.ts = asc ? pWindow->skey : pWindow->ekey;
forward = false; forward = false;
} }
@ -2168,8 +2170,8 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STable
return false; return false;
} }
if (ts == pBlockScanInfo->lastProcKey.ts) { // todo opt perf if (ts == pBlockScanInfo->lastProcKey.ts) { // todo opt perf
SRowKey nextRowKey; // lazy eval SRowKey nextRowKey; // lazy eval
tColRowGetKey(pBlockData, rowIndex, &nextRowKey); tColRowGetKey(pBlockData, rowIndex, &nextRowKey);
if (pkCompEx(&pBlockScanInfo->lastProcKey, &nextRowKey) == 0) { if (pkCompEx(&pBlockScanInfo->lastProcKey, &nextRowKey) == 0) {
return false; return false;
@ -2268,12 +2270,12 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pKeyRangeList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA; pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pKeyRangeList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA;
SRowKey* p = asc? &pScanInfo->sttRange.skey:&pScanInfo->sttRange.ekey; SRowKey* p = asc ? &pScanInfo->sttRange.skey : &pScanInfo->sttRange.ekey;
tRowKeyAssign(&pScanInfo->sttKeyInfo.nextProcKey, p); tRowKeyAssign(&pScanInfo->sttKeyInfo.nextProcKey, p);
hasData = (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA); hasData = (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA);
} else { // not clean stt blocks } else { // not clean stt blocks
INIT_KEYRANGE(&pScanInfo->sttRange); //reset the time window INIT_KEYRANGE(&pScanInfo->sttRange); // reset the time window
pScanInfo->sttBlockReturned = false; pScanInfo->sttBlockReturned = false;
hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
} }
@ -2362,8 +2364,9 @@ int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanIn
TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData};
if (IS_VAR_DATA_TYPE(pScanInfo->lastProcKey.pks[0].type)) { if (IS_VAR_DATA_TYPE(pScanInfo->lastProcKey.pks[0].type)) {
tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " pk:%s %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid, tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " pk:%s %s", pRow->pBlockData, pRow->iRow,
fRow.pBlockData->aTSKEY[fRow.iRow], pScanInfo->lastProcKey.pks[0].pData, pReader->idStr); pSttBlockReader->uid, fRow.pBlockData->aTSKEY[fRow.iRow], pScanInfo->lastProcKey.pks[0].pData,
pReader->idStr);
} }
int32_t code = int32_t code =
@ -2513,7 +2516,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
return code; return code;
} }
STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); STableBlockScanInfo* pBlockScanInfo =
getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr);
if (pBlockScanInfo == NULL) { if (pBlockScanInfo == NULL) {
goto _end; goto _end;
} }
@ -3560,7 +3564,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
int32_t order = pReader->info.order; int32_t order = pReader->info.order;
TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter);
TSDBKEY key; TSDBKEY key;
TSDBROW_INIT_KEY(pRow, key); TSDBROW_INIT_KEY(pRow, key);
if (outOfTimeWindow(key.ts, &pReader->info.window)) { if (outOfTimeWindow(key.ts, &pReader->info.window)) {
pIter->hasVal = false; pIter->hasVal = false;
@ -3572,7 +3576,8 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {
return pRow; return pRow;
} else { } else {
bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, pReader->suppInfo.numOfPks > 0); bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,
pReader->suppInfo.numOfPks > 0);
if (!dropped) { if (!dropped) {
return pRow; return pRow;
} }
@ -3597,7 +3602,8 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {
return pRow; return pRow;
} else { } else {
bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, pReader->suppInfo.numOfPks > 0); bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,
pReader->suppInfo.numOfPks > 0);
if (!dropped) { if (!dropped) {
return pRow; return pRow;
} }
@ -3606,7 +3612,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
} }
} }
int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey *pCurKey, SArray* pDelList, STsdbReader* pReader) { int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, STsdbReader* pReader) {
SRowMerger* pMerger = &pReader->status.merger; SRowMerger* pMerger = &pReader->status.merger;
while (1) { while (1) {
@ -3698,7 +3704,8 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
return code; return code;
} }
int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, STsdbReader* pReader) { int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,
STsdbReader* pReader) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SRowMerger* pMerger = &pReader->status.merger; SRowMerger* pMerger = &pReader->status.merger;
bool asc = ASCENDING_TRAVERSE(pReader->info.order); bool asc = ASCENDING_TRAVERSE(pReader->info.order);
@ -3707,8 +3714,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
pDumpInfo->rowIndex += step; pDumpInfo->rowIndex += step;
if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) {
pDumpInfo->rowIndex = pDumpInfo->rowIndex = doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, pKey, pMerger, pRange, step);
doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, pKey, pMerger, pRange, step);
} }
// all rows are consumed, let's try next file block // all rows are consumed, let's try next file block
@ -3912,8 +3918,8 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRow
static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow,
int64_t endKey, bool* freeTSRow) { int64_t endKey, bool* freeTSRow) {
TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader);
SArray* pDelList = pBlockScanInfo->delSkyline; SArray* pDelList = pBlockScanInfo->delSkyline;
uint64_t uid = pBlockScanInfo->uid; uint64_t uid = pBlockScanInfo->uid;
@ -4484,8 +4490,8 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) {
while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) {
STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p;
clearBlockScanInfo(pInfo); clearBlockScanInfo(pInfo);
// pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey.ts + step; // pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey.ts + step;
// pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey + step; // pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey + step;
} }
pStatus->uidList.currentIndex = 0; pStatus->uidList.currentIndex = 0;
@ -4815,7 +4821,6 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
return code; return code;
} }
if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) { if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) {
// prepare for the next row scan // prepare for the next row scan
int32_t step = -1; int32_t step = -1;
@ -4956,7 +4961,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
if (pResBlock->pBlockAgg == NULL) { if (pResBlock->pBlockAgg == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
for(int i = 0; i < num; ++i) { for (int i = 0; i < num; ++i) {
pResBlock->pBlockAgg[i].colId = -1; pResBlock->pBlockAgg[i].colId = -1;
} }
} }
@ -4976,6 +4981,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
} else if (pAgg->colId < pSup->colId[j]) { } else if (pAgg->colId < pSup->colId[j]) {
i += 1; i += 1;
} else if (pSup->colId[j] < pAgg->colId) { } else if (pSup->colId[j] < pAgg->colId) {
pResBlock->pBlockAgg[pSup->slotId[j]].colId = -1;
*allHave = false; *allHave = false;
j += 1; j += 1;
} }
@ -5034,9 +5040,8 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) {
SReaderStatus* pStatus = &pTReader->status; SReaderStatus* pStatus = &pTReader->status;
if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) { if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) {
// tsdbReaderSuspend2(pReader);
// tsdbReaderSuspend2(pReader); // tsdbReaderResume2(pReader);
// tsdbReaderResume2(pReader);
return pTReader->resBlockInfo.pResBlock; return pTReader->resBlockInfo.pResBlock;
} }
@ -5046,8 +5051,8 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) {
qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader);
tsdbReleaseReader(pReader); tsdbReleaseReader(pReader);
// tsdbReaderSuspend2(pReader); // tsdbReaderSuspend2(pReader);
// tsdbReaderResume2(pReader); // tsdbReaderResume2(pReader);
return ret; return ret;
} }
@ -5466,8 +5471,7 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) {
pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr; pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr;
} }
void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ }
}
void tsdbSetFilesetDelimited(STsdbReader* pReader) { pReader->bFilesetDelimited = true; } void tsdbSetFilesetDelimited(STsdbReader* pReader) { pReader->bFilesetDelimited = true; }

1
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -1103,6 +1103,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
if (vnodeValidateTableHash(pVnode, tbName) < 0) { if (vnodeValidateTableHash(pVnode, tbName) < 0) {
cRsp.code = TSDB_CODE_VND_HASH_MISMATCH; cRsp.code = TSDB_CODE_VND_HASH_MISMATCH;
taosArrayPush(rsp.pArray, &cRsp); taosArrayPush(rsp.pArray, &cRsp);
vError("vgId:%d create-table:%s failed due to hash value mismatch", TD_VID(pVnode), tbName);
continue; continue;
} }

1
source/libs/catalog/src/ctgDbg.c
View File

@ -426,6 +426,7 @@ int32_t ctgdGetClusterCacheNum(SCatalog *pCtg, int32_t type) {
break; break;
case CTG_DBG_VIEW_NUM: case CTG_DBG_VIEW_NUM:
num += ctgdGetViewNum(dbCache); num += ctgdGetViewNum(dbCache);
break;
default: default:
ctgError("invalid type:%d", type); ctgError("invalid type:%d", type);
break; break;

11
source/libs/executor/src/mergejoin.c
View File

@ -99,7 +99,7 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) {
int32_t buildEndIdx = buildGrp->endIdx; int32_t buildEndIdx = buildGrp->endIdx;
buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1; buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1;
mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft; buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx; buildGrp->endIdx = buildEndIdx;
rowsLeft = 0; rowsLeft = 0;
@ -225,7 +225,7 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
int32_t buildEndIdx = buildGrp->endIdx; int32_t buildEndIdx = buildGrp->endIdx;
buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1; buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1;
mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft; buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx; buildGrp->endIdx = buildEndIdx;
rowsLeft = 0; rowsLeft = 0;
@ -291,7 +291,6 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft; buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx; buildGrp->endIdx = buildEndIdx;
rowsLeft = 0;
break; break;
} }
@ -682,7 +681,7 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) {
int32_t buildEndIdx = buildGrp->endIdx; int32_t buildEndIdx = buildGrp->endIdx;
buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1; buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1;
mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft; buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx; buildGrp->endIdx = buildEndIdx;
rowsLeft = 0; rowsLeft = 0;
@ -1402,7 +1401,6 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft; buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx; buildGrp->endIdx = buildEndIdx;
rowsLeft = 0;
break; break;
} }
@ -1712,7 +1710,6 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
buildGrp->readIdx += rowsLeft; buildGrp->readIdx += rowsLeft;
buildGrp->endIdx = buildEndIdx; buildGrp->endIdx = buildEndIdx;
rowsLeft = 0;
break; break;
} }
@ -2004,7 +2001,7 @@ int32_t mAsofBackwardDumpGrpCache(SMJoinWindowCtx* pCtx) {
} }
buildGrp.endIdx = buildGrp.readIdx + rowsLeft - 1; buildGrp.endIdx = buildGrp.readIdx + rowsLeft - 1;
mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, &buildGrp); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, &buildGrp));
pCtx->cache.outRowIdx += rowsLeft; pCtx->cache.outRowIdx += rowsLeft;
break; break;
} }

2
source/libs/executor/src/mergeoperator.c
View File

@ -182,7 +182,7 @@ SSDataBlock* doSortMerge(SOperatorInfo* pOperator) {
resetLimitInfoForNextGroup(&pInfo->limitInfo); resetLimitInfoForNextGroup(&pInfo->limitInfo);
} }
applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo); (void)applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo);
if (p->info.rows > 0) { if (p->info.rows > 0) {
break; break;

6
source/libs/executor/src/scanoperator.c
View File

@ -3902,17 +3902,19 @@ static int32_t openSubTablesMergeSort(STmsSubTablesMergeInfo* pSubTblsInfo) {
if (pInput->rowIdx == -1) { if (pInput->rowIdx == -1) {
continue; continue;
} }
if (pInput->type == SUB_TABLE_MEM_BLOCK) { if (pInput->type == SUB_TABLE_MEM_BLOCK) {
pInput->rowIdx = 0; pInput->rowIdx = 0;
pInput->pageIdx = -1; pInput->pageIdx = -1;
} }
pInput->pInputBlock = (pInput->type == SUB_TABLE_MEM_BLOCK) ? pInput->pReaderBlock : pInput->pPageBlock; pInput->pInputBlock = (pInput->type == SUB_TABLE_MEM_BLOCK) ? pInput->pReaderBlock : pInput->pPageBlock;
SColumnInfoData* col = taosArrayGet(pInput->pInputBlock->pDataBlock, pSubTblsInfo->pTsOrderInfo->slotId); SColumnInfoData* col = taosArrayGet(pInput->pInputBlock->pDataBlock, pSubTblsInfo->pTsOrderInfo->slotId);
pInput->aTs = (int64_t*)col->pData; pInput->aTs = (int64_t*)col->pData;
} }
__merge_compare_fn_t mergeCompareFn = (!pSubTblsInfo->pPkOrderInfo) ? subTblRowCompareTsFn : subTblRowCompareTsPkFn; __merge_compare_fn_t mergeCompareFn = (!pSubTblsInfo->pPkOrderInfo) ? subTblRowCompareTsFn : subTblRowCompareTsPkFn;
tMergeTreeCreate(&pSubTblsInfo->pTree, pSubTblsInfo->numSubTables, pSubTblsInfo, mergeCompareFn); return tMergeTreeCreate(&pSubTblsInfo->pTree, pSubTblsInfo->numSubTables, pSubTblsInfo, mergeCompareFn);
return TSDB_CODE_SUCCESS;
} }
static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) { static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) {

53
source/libs/parser/src/parInsertStmt.c
View File

@ -51,7 +51,8 @@ int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo) { int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx,
SStbInterlaceInfo* pBuildInfo) {
// merge according to vgId // merge according to vgId
return insAppendStmtTableDataCxt(pAllVgHash, pTbData, pTbCtx, pBuildInfo); return insAppendStmtTableDataCxt(pAllVgHash, pTbData, pTbCtx, pBuildInfo);
} }
@ -70,18 +71,16 @@ int32_t qBuildStmtFinOutput(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDat
return code; return code;
} }
/* /*
int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx, int32_t tbNum) { int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx, int32_t
int32_t code = TSDB_CODE_SUCCESS; tbNum) { int32_t code = TSDB_CODE_SUCCESS; SArray* pVgDataBlocks = NULL; SVnodeModifyOpStmt*
SArray* pVgDataBlocks = NULL; pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
// merge according to vgId // merge according to vgId
if (tbNum > 0) { if (tbNum > 0) {
code = insMergeStmtTableDataCxt(pTbCtx, pBlockList, &pVgDataBlocks, true, tbNum); code = insMergeStmtTableDataCxt(pTbCtx, pBlockList, &pVgDataBlocks, true, tbNum);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks); code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks);
} }
@ -102,7 +101,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash
if (taosHashGetSize(pBlockHash) > 0) { if (taosHashGetSize(pBlockHash) > 0) {
code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks, true); code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks, true);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks, false); code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks, false);
} }
@ -217,6 +216,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName,
pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL);
pTag = NULL;
end: end:
for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
@ -227,6 +227,7 @@ end:
} }
taosArrayDestroy(pTagArray); taosArrayDestroy(pTagArray);
taosArrayDestroy(tagName); taosArrayDestroy(tagName);
taosMemoryFree(pTag);
return code; return code;
} }
@ -243,7 +244,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
if (NULL == dst->length) { if (NULL == dst->length) {
dst->length = taosMemoryRealloc(dst->length, sizeof(int32_t) * src->num); dst->length = taosMemoryRealloc(dst->length, sizeof(int32_t) * src->num);
if (NULL == dst->buffer) { if (NULL == dst->length) {
taosMemoryFreeClear(dst->buffer); taosMemoryFreeClear(dst->buffer);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -276,7 +277,8 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, STSchema** pTSchema, SBindInfo* pBindInfos) { int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo* pBindInfos) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -293,20 +295,21 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind
} }
for (int c = 0; c < boundInfo->numOfBound; ++c) { for (int c = 0; c < boundInfo->numOfBound; ++c) {
SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]];
if (pColSchema->colId <= lastColId) { if (pColSchema->colId <= lastColId) {
colInOrder = false; colInOrder = false;
} else { } else {
lastColId = pColSchema->colId; lastColId = pColSchema->colId;
} }
//SColData* pCol = taosArrayGet(pCols, c); // SColData* pCol = taosArrayGet(pCols, c);
if (bind[c].num != rowNum) { if (bind[c].num != rowNum) {
code = buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); code = buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
goto _return; goto _return;
} }
if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) &&
bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type
code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
goto _return; goto _return;
} }
@ -325,10 +328,10 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind
pBindInfos[c].bind = pBind; pBindInfos[c].bind = pBind;
pBindInfos[c].type = pColSchema->type; pBindInfos[c].type = pColSchema->type;
//code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); // code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes -
//if (code) { // VARSTR_HEADER_SIZE: -1); if (code) {
// goto _return; // goto _return;
//} // }
} }
code = tRowBuildFromBind(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols); code = tRowBuildFromBind(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols);
@ -362,7 +365,8 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
goto _return; goto _return;
} }
if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) &&
bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type
code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
goto _return; goto _return;
} }
@ -377,7 +381,8 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
pBind = bind + c; pBind = bind + c;
} }
code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); code = tColDataAddValueByBind(pCol, pBind,
IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1);
if (code) { if (code) {
goto _return; goto _return;
} }
@ -393,9 +398,8 @@ _return:
return code; return code;
} }
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t colIdx, int32_t rowNum) {
int32_t rowNum) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -510,7 +514,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel
} }
int32_t qResetStmtColumns(SArray* pCols, bool deepClear) { int32_t qResetStmtColumns(SArray* pCols, bool deepClear) {
int32_t colNum = taosArrayGetSize(pCols); int32_t colNum = taosArrayGetSize(pCols);
for (int32_t i = 0; i < colNum; ++i) { for (int32_t i = 0; i < colNum; ++i) {
SColData* pCol = (SColData*)taosArrayGet(pCols, i); SColData* pCol = (SColData*)taosArrayGet(pCols, i);
@ -524,7 +528,6 @@ int32_t qResetStmtColumns(SArray* pCols, bool deepClear) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) { int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) {
STableDataCxt* pBlock = (STableDataCxt*)block; STableDataCxt* pBlock = (STableDataCxt*)block;
int32_t colNum = taosArrayGetSize(pBlock->pData->aCol); int32_t colNum = taosArrayGetSize(pBlock->pData->aCol);
@ -589,7 +592,7 @@ int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool rese
pNewTb->pCreateTbReq = NULL; pNewTb->pCreateTbReq = NULL;
pNewTb->aCol = taosArrayDup(pCxt->pData->aCol, NULL); pNewTb->aCol = taosArrayDup(pCxt->pData->aCol, NULL);
if (NULL == pNewTb) { if (NULL == pNewTb->aCol) {
insDestroyTableDataCxt(*pDst); insDestroyTableDataCxt(*pDst);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }

79
source/libs/parser/src/parInsertUtil.c
View File

@ -115,8 +115,7 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c
if (dbName == NULL) { if (dbName == NULL) {
return buildInvalidOperationMsg(pMsgBuf, msg3); return buildInvalidOperationMsg(pMsgBuf, msg3);
} }
if (name[0] == '\0') if (name[0] == '\0') return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, msg4);
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, msg4);
code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); code = tNameSetDbName(pName, acctId, dbName, strlen(dbName));
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -487,12 +486,12 @@ int insColDataComp(const void* lp, const void* rp) {
return 0; return 0;
} }
int32_t insTryAddTableVgroupInfo(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, int32_t* vgId,
int32_t insTryAddTableVgroupInfo(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, int32_t* vgId, STableColsData* pTbData, SName* sname) { STableColsData* pTbData, SName* sname) {
if (*vgId >= 0 && taosHashGet(pAllVgHash, (const char*)vgId, sizeof(*vgId))) { if (*vgId >= 0 && taosHashGet(pAllVgHash, (const char*)vgId, sizeof(*vgId))) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
SRequestConnInfo conn = {.pTrans = pBuildInfo->transport, SRequestConnInfo conn = {.pTrans = pBuildInfo->transport,
.requestId = pBuildInfo->requestId, .requestId = pBuildInfo->requestId,
@ -503,23 +502,23 @@ int32_t insTryAddTableVgroupInfo(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuild
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
code = taosHashPut(pAllVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo)); code = taosHashPut(pAllVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo));
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, STableColsData* pTbData,
int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, STableColsData* pTbData, uint64_t* uid, int32_t* vgId) { uint64_t* uid, int32_t* vgId) {
STableVgUid* pTbInfo = NULL; STableVgUid* pTbInfo = NULL;
int32_t code = 0; int32_t code = 0;
if (pTbData->getFromHash) { if (pTbData->getFromHash) {
pTbInfo = (STableVgUid*)tSimpleHashGet(pBuildInfo->pTableHash, pTbData->tbName, strlen(pTbData->tbName)); pTbInfo = (STableVgUid*)tSimpleHashGet(pBuildInfo->pTableHash, pTbData->tbName, strlen(pTbData->tbName));
} }
if (NULL == pTbInfo) { if (NULL == pTbInfo) {
SName sname; SName sname;
@ -540,7 +539,7 @@ int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
*uid = pTableMeta->uid; *uid = pTableMeta->uid;
*vgId = pTableMeta->vgId; *vgId = pTableMeta->vgId;
@ -548,7 +547,7 @@ int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo
tSimpleHashPut(pBuildInfo->pTableHash, pTbData->tbName, strlen(pTbData->tbName), &tbInfo, sizeof(tbInfo)); tSimpleHashPut(pBuildInfo->pTableHash, pTbData->tbName, strlen(pTbData->tbName), &tbInfo, sizeof(tbInfo));
code = insTryAddTableVgroupInfo(pAllVgHash, pBuildInfo, vgId, pTbData, &sname); code = insTryAddTableVgroupInfo(pAllVgHash, pBuildInfo, vgId, pTbData, &sname);
taosMemoryFree(pTableMeta); taosMemoryFree(pTableMeta);
} else { } else {
*uid = pTbInfo->uid; *uid = pTbInfo->uid;
@ -558,7 +557,6 @@ int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo
return code; return code;
} }
int32_t qBuildStmtFinOutput1(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDataBlocks) { int32_t qBuildStmtFinOutput1(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDataBlocks) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
@ -570,20 +568,19 @@ int32_t qBuildStmtFinOutput1(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDa
return code; return code;
} }
int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx,
SStbInterlaceInfo* pBuildInfo) {
int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo) { int32_t code = TSDB_CODE_SUCCESS;
int32_t code = TSDB_CODE_SUCCESS;
uint64_t uid; uint64_t uid;
int32_t vgId; int32_t vgId;
pTbCtx->pData->aRowP = pTbData->aCol; pTbCtx->pData->aRowP = pTbData->aCol;
code = insGetStmtTableVgUid(pAllVgHash, pBuildInfo, pTbData, &uid, &vgId); code = insGetStmtTableVgUid(pAllVgHash, pBuildInfo, pTbData, &uid, &vgId);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
pTbCtx->pMeta->vgId = vgId; pTbCtx->pMeta->vgId = vgId;
pTbCtx->pMeta->uid = uid; pTbCtx->pMeta->uid = uid;
pTbCtx->pData->uid = uid; pTbCtx->pData->uid = uid;
@ -611,24 +608,24 @@ int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData,
} else { } else {
pVgCxt = *(SVgroupDataCxt**)pp; pVgCxt = *(SVgroupDataCxt**)pp;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = fillVgroupDataCxt(pTbCtx, pVgCxt, false, false); code = fillVgroupDataCxt(pTbCtx, pVgCxt, false, false);
} }
if (taosArrayGetSize(pVgCxt->pData->aSubmitTbData) >= 20000) { if (taosArrayGetSize(pVgCxt->pData->aSubmitTbData) >= 20000) {
code = qBuildStmtFinOutput1((SQuery*)pBuildInfo->pQuery, pAllVgHash, pBuildInfo->pVgroupList); code = qBuildStmtFinOutput1((SQuery*)pBuildInfo->pQuery, pAllVgHash, pBuildInfo->pVgroupList);
//taosArrayClear(pVgCxt->pData->aSubmitTbData); // taosArrayClear(pVgCxt->pData->aSubmitTbData);
tDestroySubmitReq(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); tDestroySubmitReq(pVgCxt->pData, TSDB_MSG_FLG_ENCODE);
//insDestroyVgroupDataCxt(pVgCxt); // insDestroyVgroupDataCxt(pVgCxt);
} }
return code; return code;
} }
/* /*
int32_t insMergeStmtTableDataCxt(STableDataCxt* pTableCxt, SArray* pTableList, SArray** pVgDataBlocks, bool isRebuild, int32_t tbNum) { int32_t insMergeStmtTableDataCxt(STableDataCxt* pTableCxt, SArray* pTableList, SArray** pVgDataBlocks, bool isRebuild,
SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); int32_t tbNum) { SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES); SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES);
if (NULL == pVgroupHash || NULL == pVgroupList) { if (NULL == pVgroupHash || NULL == pVgroupList) {
taosHashCleanup(pVgroupHash); taosHashCleanup(pVgroupHash);
@ -644,7 +641,7 @@ int32_t insMergeStmtTableDataCxt(STableDataCxt* pTableCxt, SArray* pTableList, S
pTableCxt->pMeta->uid = pTableCols->uid; pTableCxt->pMeta->uid = pTableCols->uid;
pTableCxt->pData->uid = pTableCols->uid; pTableCxt->pData->uid = pTableCols->uid;
pTableCxt->pData->aCol = pTableCols->aCol; pTableCxt->pData->aCol = pTableCols->aCol;
SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0); SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0);
if (pCol->nVal <= 0) { if (pCol->nVal <= 0) {
continue; continue;
@ -810,7 +807,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList,
if (taosArrayGetSize(src->pData->aSubmitTbData) <= 0) { if (taosArrayGetSize(src->pData->aSubmitTbData) <= 0) {
continue; continue;
} }
SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
if (NULL == dst) { if (NULL == dst) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
} }
@ -875,7 +872,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
} }
char* p = (char*)data; char* p = (char*)data;
// | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each column length | // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
// column length |
int32_t version = *(int32_t*)data; int32_t version = *(int32_t*)data;
p += sizeof(int32_t); p += sizeof(int32_t);
p += sizeof(int32_t); p += sizeof(int32_t);
@ -890,7 +888,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
p += sizeof(uint64_t); p += sizeof(uint64_t);
int8_t* fields = p; int8_t* fields = p;
if(*fields >= TSDB_DATA_TYPE_MAX || *fields < 0){ if (*fields >= TSDB_DATA_TYPE_MAX || *fields < 0) {
uError("fields type error:%d", *fields); uError("fields type error:%d", *fields);
ret = TSDB_CODE_INVALID_PARA; ret = TSDB_CODE_INVALID_PARA;
goto end; goto end;
@ -911,7 +909,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
goto end; goto end;
} }
if (tFields != NULL && numFields > boundInfo->numOfBound) { if (tFields != NULL && numFields > boundInfo->numOfBound) {
if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound); if (errstr != NULL)
snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
ret = TSDB_CODE_INVALID_PARA; ret = TSDB_CODE_INVALID_PARA;
goto end; goto end;
} }
@ -920,8 +919,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema* pColSchema = &pSchema[j]; SSchema* pColSchema = &pSchema[j];
SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j);
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", if (errstr != NULL)
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t))); snprintf(errstr, errstrLen,
"column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name,
*(int32_t*)(fields + sizeof(int8_t)));
ret = TSDB_CODE_INVALID_PARA; ret = TSDB_CODE_INVALID_PARA;
goto end; goto end;
} }
@ -951,8 +953,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema* pColSchema = &pSchema[j]; SSchema* pColSchema = &pSchema[j];
if (strcmp(pColSchema->name, tFields[i].name) == 0) { if (strcmp(pColSchema->name, tFields[i].name) == 0) {
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", if (errstr != NULL)
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t))); snprintf(errstr, errstrLen,
"column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name,
*(int32_t*)(fields + sizeof(int8_t)));
ret = TSDB_CODE_INVALID_PARA; ret = TSDB_CODE_INVALID_PARA;
goto end; goto end;
} }
@ -986,7 +991,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
} }
} }
if(!hasTs){ if (!hasTs) {
if (errstr != NULL) snprintf(errstr, errstrLen, "timestamp column(primary key) not found in raw data"); if (errstr != NULL) snprintf(errstr, errstrLen, "timestamp column(primary key) not found in raw data");
ret = TSDB_CODE_INVALID_PARA; ret = TSDB_CODE_INVALID_PARA;
goto end; goto end;

8
source/libs/parser/src/parTranslater.c
View File

@ -8112,9 +8112,11 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt,
for (int32_t i = 1; i < num; ++i) { for (int32_t i = 1; i < num; ++i) {
SRetention* pRetension = taosArrayGet(dbCfg.pRetensions, i); SRetention* pRetension = taosArrayGet(dbCfg.pRetensions, i);
STranslateContext cxt = {0}; STranslateContext cxt = {0};
initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt); code = initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt);
code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, 1 == i ? &pReq->pAst1 : &pReq->pAst2, if (TSDB_CODE_SUCCESS == code) {
1 == i ? &pReq->ast1Len : &pReq->ast2Len); code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, 1 == i ? &pReq->pAst1 : &pReq->pAst2,
1 == i ? &pReq->ast1Len : &pReq->ast2Len);
}
destroyTranslateContext(&cxt); destroyTranslateContext(&cxt);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
break; break;

52
source/libs/qworker/src/qworker.c
View File

@ -90,7 +90,6 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re
QW_RET(TSDB_CODE_SUCCESS); QW_RET(TSDB_CODE_SUCCESS);
} }
int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
qTaskInfo_t taskHandle = ctx->taskHandle; qTaskInfo_t taskHandle = ctx->taskHandle;
@ -192,12 +191,12 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
} else { } else {
QW_TASK_DLOG("dyn task qExecTask done, useconds:%" PRIu64, useconds); QW_TASK_DLOG("dyn task qExecTask done, useconds:%" PRIu64, useconds);
} }
ctx->queryExecDone = true; ctx->queryExecDone = true;
} }
dsEndPut(sinkHandle, useconds); dsEndPut(sinkHandle, useconds);
if (queryStop) { if (queryStop) {
*queryStop = true; *queryStop = true;
} }
@ -240,7 +239,6 @@ bool qwTaskNotInExec(SQWTaskCtx *ctx) {
return false; return false;
} }
int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) {
int32_t taskNum = 0; int32_t taskNum = 0;
@ -324,7 +322,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
if (!ctx->dynamicTask) { if (!ctx->dynamicTask) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask);
} }
if (NULL == pRsp) { if (NULL == pRsp) {
QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, len, &pRsp)); QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, len, &pRsp));
*pOutput = output; *pOutput = output;
@ -351,8 +349,8 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
// set the serialize start position // set the serialize start position
output.pData = pRsp->data + *dataLen - (len + PAYLOAD_PREFIX_LEN); output.pData = pRsp->data + *dataLen - (len + PAYLOAD_PREFIX_LEN);
((int32_t*) output.pData)[0] = len; ((int32_t *)output.pData)[0] = len;
((int32_t*) output.pData)[1] = rawLen; ((int32_t *)output.pData)[1] = rawLen;
output.pData += sizeof(int32_t) * 2; output.pData += sizeof(int32_t) * 2;
code = dsGetDataBlock(ctx->sinkHandle, &output); code = dsGetDataBlock(ctx->sinkHandle, &output);
@ -439,8 +437,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg, int32_t code) {
int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, int32_t code) {
if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL) { if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL) {
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
void *rsp = NULL; void *rsp = NULL;
@ -466,13 +463,13 @@ int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, i
qwMsg->connInfo = ctx->dataConnInfo; qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code); qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
rsp = NULL; rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
dataLen); dataLen);
} }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -487,14 +484,14 @@ int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg
if (!atomic_val_compare_exchange_8((int8_t*)&ctx->queryEnd, true, false)) { if (!atomic_val_compare_exchange_8((int8_t*)&ctx->queryEnd, true, false)) {
QW_TASK_ELOG("dynamic task prev exec not finished, queryEnd:%d", ctx->queryEnd); QW_TASK_ELOG("dynamic task prev exec not finished, queryEnd:%d", ctx->queryEnd);
return TSDB_CODE_ACTION_IN_PROGRESS; return TSDB_CODE_ACTION_IN_PROGRESS;
} }
#else #else
ctx->queryExecDone = false; ctx->queryExecDone = false;
ctx->queryEnd = false; ctx->queryEnd = false;
#endif #endif
dsReset(ctx->sinkHandle); dsReset(ctx->sinkHandle);
qUpdateOperatorParam(ctx->taskHandle, qwMsg->msg); qUpdateOperatorParam(ctx->taskHandle, qwMsg->msg);
QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH);
@ -511,7 +508,6 @@ int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) {
int32_t code = 0; int32_t code = 0;
SQWTaskCtx *ctx = NULL; SQWTaskCtx *ctx = NULL;
@ -747,7 +743,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
QW_ERR_JRET(code); QW_ERR_JRET(code);
} }
code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql, OPTR_EXEC_MODEL_BATCH); code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql,
OPTR_EXEC_MODEL_BATCH);
sql = NULL; sql = NULL;
if (code) { if (code) {
QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code));
@ -772,7 +769,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
ctx->queryExecDone = true; ctx->queryExecDone = true;
ctx->queryEnd = true; ctx->queryEnd = true;
} }
_return: _return:
taosMemoryFree(sql); taosMemoryFree(sql);
@ -809,7 +806,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
if (!queryStop) { if (!queryStop) {
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop)); QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop));
} }
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
SOutputData sOutput = {0}; SOutputData sOutput = {0};
QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput)); QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput));
@ -878,9 +875,9 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
} }
int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
int32_t code = 0; int32_t code = 0;
int32_t dataLen = 0; int32_t dataLen = 0;
int32_t rawDataLen = 0; int32_t rawDataLen = 0;
bool locked = false; bool locked = false;
SQWTaskCtx *ctx = NULL; SQWTaskCtx *ctx = NULL;
@ -962,8 +959,8 @@ _return:
qwFreeFetchRsp(rsp); qwFreeFetchRsp(rsp);
rsp = NULL; rsp = NULL;
} }
} else { } else {
//qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code); // qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code);
} }
QW_RET(TSDB_CODE_SUCCESS); QW_RET(TSDB_CODE_SUCCESS);
@ -1005,6 +1002,8 @@ _return:
if (ctx) { if (ctx) {
QW_UPDATE_RSP_CODE(ctx, code); QW_UPDATE_RSP_CODE(ctx, code);
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask);
} else {
tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER);
} }
} }
@ -1013,6 +1012,10 @@ _return:
} }
if (ctx) { if (ctx) {
if (qwMsg->connInfo.handle != ctx->ctrlConnInfo.handle) {
tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER);
}
qwReleaseTaskCtx(mgmt, ctx); qwReleaseTaskCtx(mgmt, ctx);
} }
@ -1040,7 +1043,7 @@ int32_t qwProcessNotify(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
if (ctx->explain && !ctx->explainRsped) { if (ctx->explain && !ctx->explainRsped) {
QW_ERR_RET(qwSendExplainResponse(QW_FPARAMS(), ctx)); QW_ERR_RET(qwSendExplainResponse(QW_FPARAMS(), ctx));
} }
break; break;
default: default:
QW_ELOG("Invalid task notify type %d", qwMsg->msgType); QW_ELOG("Invalid task notify type %d", qwMsg->msgType);
QW_ERR_JRET(TSDB_CODE_INVALID_MSG); QW_ERR_JRET(TSDB_CODE_INVALID_MSG);
@ -1067,7 +1070,6 @@ _return:
QW_RET(TSDB_CODE_SUCCESS); QW_RET(TSDB_CODE_SUCCESS);
} }
int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
int32_t code = 0; int32_t code = 0;
SSchedulerHbRsp rsp = {0}; SSchedulerHbRsp rsp = {0};

8
source/libs/stream/inc/streamInt.h
View File

@ -55,10 +55,10 @@ struct SActiveCheckpointInfo {
int64_t activeId; // current active checkpoint id int64_t activeId; // current active checkpoint id
int64_t failedId; int64_t failedId;
bool dispatchTrigger; bool dispatchTrigger;
SArray* pDispatchTriggerList; // SArray<STaskTriggerSendInfo> SArray* pDispatchTriggerList; // SArray<STaskTriggerSendInfo>
SArray* pReadyMsgList; // SArray<STaskCheckpointReadyInfo*> SArray* pReadyMsgList; // SArray<STaskCheckpointReadyInfo*>
int8_t allUpstreamTriggerRecv; int8_t allUpstreamTriggerRecv;
SArray* pCheckpointReadyRecvList; // SArray<STaskDownstreamReadyInfo> SArray* pCheckpointReadyRecvList; // SArray<STaskDownstreamReadyInfo>
int32_t checkCounter; int32_t checkCounter;
tmr_h pChkptTriggerTmr; tmr_h pChkptTriggerTmr;
int32_t sendReadyCheckCounter; int32_t sendReadyCheckCounter;
@ -100,7 +100,7 @@ typedef struct {
int32_t upstreamNodeId; int32_t upstreamNodeId;
int32_t transId; int32_t transId;
int32_t childId; int32_t childId;
SRpcMsg msg; // for mnode checkpoint-source rsp SRpcMsg msg; // for mnode checkpoint-source rsp
int64_t checkpointId; int64_t checkpointId;
int64_t recvTs; int64_t recvTs;
int32_t sendCompleted; int32_t sendCompleted;

39
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -140,7 +140,7 @@ int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest);
int32_t valueToString(void* k, char* buf); int32_t valueToString(void* k, char* buf);
int32_t valueIsStale(void* k, int64_t ts); int32_t valueIsStale(void* k, int64_t ts);
void destroyCompare(void* arg); void destroyCompare(void* arg);
static void cleanDir(const char* pPath, const char* id); static void cleanDir(const char* pPath, const char* id);
static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len);
@ -194,9 +194,7 @@ int32_t getCfIdx(const char* cfName) {
return idx; return idx;
} }
bool isValidCheckpoint(const char* dir) { bool isValidCheckpoint(const char* dir) { return true; }
return true;
}
int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) {
// impl later // impl later
@ -486,9 +484,7 @@ _ERROR:
return code; return code;
} }
int32_t backendCopyFiles(const char* src, const char* dst) { int32_t backendCopyFiles(const char* src, const char* dst) { return backendFileCopyFilesImpl(src, dst); }
return backendFileCopyFilesImpl(src, dst);
}
static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* checkpointPath, int64_t checkpointId, static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* checkpointPath, int64_t checkpointId,
const char* defaultPath) { const char* defaultPath) {
@ -540,7 +536,8 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId
char* chkptPath = taosMemoryCalloc(1, pathLen); char* chkptPath = taosMemoryCalloc(1, pathLen);
if (chkptId > 0) { if (chkptId > 0) {
snprintf(chkptPath, pathLen, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkptId); snprintf(chkptPath, pathLen, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint",
chkptId);
code = rebuildFromLocalCheckpoint(key, chkptPath, chkptId, defaultPath); code = rebuildFromLocalCheckpoint(key, chkptPath, chkptId, defaultPath);
if (code != 0) { if (code != 0) {
@ -549,11 +546,12 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId
if (code != 0) { if (code != 0) {
stError("failed to start stream backend at %s, reason: %s, restart from default defaultPath:%s", chkptPath, stError("failed to start stream backend at %s, reason: %s, restart from default defaultPath:%s", chkptPath,
tstrerror(code), defaultPath); tstrerror(code), defaultPath);
code = 0; // reset the error code code = 0; // reset the error code
} }
} else { // no valid checkpoint id } else { // no valid checkpoint id
stInfo("%s no valid checkpoint ever generated, no need to copy checkpoint data, clean defaultPath:%s", key, defaultPath); stInfo("%s no valid checkpoint ever generated, no need to copy checkpoint data, clean defaultPath:%s", key,
defaultPath);
cleanDir(defaultPath, key); cleanDir(defaultPath, key);
} }
@ -2143,13 +2141,18 @@ void taskDbDestroy(void* pDb, bool flush) {
rocksdb_flushoptions_destroy(flushOpt); rocksdb_flushoptions_destroy(flushOpt);
} }
} }
for (int i = 0; i < nCf; i++) {
if (wrapper->pCf[i] != NULL) { if (wrapper->pCf != NULL) {
rocksdb_column_family_handle_destroy(wrapper->pCf[i]); for (int i = 0; i < nCf; i++) {
if (wrapper->pCf[i] != NULL) {
rocksdb_column_family_handle_destroy(wrapper->pCf[i]);
}
} }
} }
if (wrapper->db) rocksdb_close(wrapper->db); if (wrapper->db) {
rocksdb_close(wrapper->db);
}
rocksdb_options_destroy(wrapper->dbOpt); rocksdb_options_destroy(wrapper->dbOpt);
rocksdb_readoptions_destroy(wrapper->readOpt); rocksdb_readoptions_destroy(wrapper->readOpt);
@ -2205,7 +2208,8 @@ int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char
return code; return code;
} }
int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path, SArray* list, const char* idStr) { int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path, SArray* list,
const char* idStr) {
int32_t code = 0; int32_t code = 0;
SBkdMgt* p = (SBkdMgt*)bkdChkpMgt; SBkdMgt* p = (SBkdMgt*)bkdChkpMgt;
@ -2224,7 +2228,8 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64
return code; return code;
} }
int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list, const char* idStr) { int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list,
const char* idStr) {
int32_t code = -1; int32_t code = -1;
STaskDbWrapper* pDb = arg; STaskDbWrapper* pDb = arg;
ECHECKPOINT_BACKUP_TYPE utype = type; ECHECKPOINT_BACKUP_TYPE utype = type;

18
source/libs/stream/src/streamCheckStatus.c
View File

@ -58,7 +58,8 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
} }
if (pInfo->stage < stage) { if (pInfo->stage < stage) {
stError("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), new stage received:%" PRId64 ", prev:%" PRId64, stError("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), new stage received:%" PRId64
", prev:%" PRId64,
id, upstreamTaskId, vgId, stage, pInfo->stage); id, upstreamTaskId, vgId, stage, pInfo->stage);
// record the checkpoint failure id and sent to mnode // record the checkpoint failure id and sent to mnode
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
@ -70,7 +71,6 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
} }
if (pInfo->stage != stage) { if (pInfo->stage != stage) {
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask)->state; ETaskStatus status = streamTaskGetStatus(pTask)->state;
if (status == TASK_STATUS__CK) { if (status == TASK_STATUS__CK) {
@ -168,7 +168,8 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS
} else { } else {
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId);
if (pTask != NULL) { if (pTask != NULL) {
pRsp->status = streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); pRsp->status =
streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage);
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState* pState = streamTaskGetStatus(pTask);
stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64 stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64
@ -183,7 +184,6 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS
pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status);
} }
} }
} }
int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
@ -402,7 +402,8 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId);
if (p != NULL) { if (p != NULL) {
if (reqId != p->reqId) { if (reqId != p->reqId) {
stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 " expired check-rsp recv from downstream task:0x%x, discarded", stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64
" expired check-rsp recv from downstream task:0x%x, discarded",
id, reqId, p->reqId, taskId); id, reqId, p->reqId, taskId);
taosThreadMutexUnlock(&pInfo->checkInfoLock); taosThreadMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -435,7 +436,7 @@ int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) {
ASSERT(pInfo->startTs > 0); ASSERT(pInfo->startTs > 0);
stError("s-task:%s already in check procedure, checkTs:%" PRId64 ", start monitor check rsp failed", id, stError("s-task:%s already in check procedure, checkTs:%" PRId64 ", start monitor check rsp failed", id,
pInfo->startTs); pInfo->startTs);
pInfo->stopCheckProcess = 0; // disable auto stop of check process pInfo->stopCheckProcess = 0; // disable auto stop of check process
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -509,8 +510,8 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher; STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher;
setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId); setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id,
id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId);
streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet);
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
@ -760,4 +761,3 @@ void rspMonitorFn(void* param, void* tmrId) {
taosArrayDestroy(pNotReadyList); taosArrayDestroy(pNotReadyList);
taosArrayDestroy(pTimeoutList); taosArrayDestroy(pTimeoutList);
} }

75
source/libs/stream/src/streamCheckpoint.c
View File

@ -24,11 +24,13 @@ static int32_t streamTaskUploadCheckpoint(const char* id, const char* path);
static int32_t deleteCheckpoint(const char* id); static int32_t deleteCheckpoint(const char* id);
static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName); static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName);
static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask); static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask);
static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId); static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId,
int32_t transId);
static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList); static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList);
static void checkpointTriggerMonitorFn(void* param, void* tmrId); static void checkpointTriggerMonitorFn(void* param, void* tmrId);
static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId); static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId,
int32_t transId);
SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId,
int32_t transId) { int32_t transId) {
@ -96,7 +98,7 @@ int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTri
if (pRsp->rspCode != TSDB_CODE_SUCCESS) { if (pRsp->rspCode != TSDB_CODE_SUCCESS) {
stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", pTask->id.idStr, stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", pTask->id.idStr,
pRsp->upstreamTaskId, tstrerror(pRsp->rspCode)); pRsp->upstreamTaskId, tstrerror(pRsp->rspCode));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -108,7 +110,7 @@ int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId
SRpcHandleInfo* pRpcInfo, int32_t code) { SRpcHandleInfo* pRpcInfo, int32_t code) {
int32_t size = sizeof(SMsgHead) + sizeof(SCheckpointTriggerRsp); int32_t size = sizeof(SMsgHead) + sizeof(SCheckpointTriggerRsp);
void* pBuf = rpcMallocCont(size); void* pBuf = rpcMallocCont(size);
SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
((SMsgHead*)pBuf)->vgId = htonl(downstreamNodeId); ((SMsgHead*)pBuf)->vgId = htonl(downstreamNodeId);
@ -162,15 +164,15 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
if (pTask->chkInfo.checkpointId > checkpointId) { if (pTask->chkInfo.checkpointId > checkpointId) {
stError("s-task:%s vgId:%d current checkpointId:%" PRId64 stError("s-task:%s vgId:%d current checkpointId:%" PRId64
" recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard",
id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId);
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (pTask->chkInfo.checkpointId == checkpointId) { if (pTask->chkInfo.checkpointId == checkpointId) {
{ // send checkpoint-ready msg to upstream { // send checkpoint-ready msg to upstream
SRpcMsg msg ={0}; SRpcMsg msg = {0};
SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId); SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId);
initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg);
@ -362,7 +364,8 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
taosThreadMutexUnlock(&pInfo->lock); taosThreadMutexUnlock(&pInfo->lock);
if (notReady == 0) { if (notReady == 0) {
stDebug("s-task:%s all downstream task(s) have completed build checkpoint, start to do checkpoint for current task", id); stDebug("s-task:%s all downstream task(s) have completed build checkpoint, start to do checkpoint for current task",
id);
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId); appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId);
} }
@ -371,11 +374,11 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId) { int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
int32_t numOfConfirmed = 0; int32_t numOfConfirmed = 0;
taosThreadMutexLock(&pInfo->lock); taosThreadMutexLock(&pInfo->lock);
for(int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) {
STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i);
if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) {
pReadyInfo->sendCompleted = 1; pReadyInfo->sendCompleted = 1;
@ -385,7 +388,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream
} }
} }
for(int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) {
STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i);
if (pReadyInfo->sendCompleted == 1) { if (pReadyInfo->sendCompleted == 1) {
numOfConfirmed += 1; numOfConfirmed += 1;
@ -443,13 +446,14 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
SStreamTaskState* pStatus = streamTaskGetStatus(pTask); SStreamTaskState* pStatus = streamTaskGetStatus(pTask);
// if (restored && (pStatus->state != TASK_STATUS__CK)) { // if (restored && (pStatus->state != TASK_STATUS__CK)) {
// stDebug("s-task:0x%x vgId:%d restored:%d status:%s not update checkpoint-info, checkpointId:%" PRId64 "->%" PRId64 // stDebug("s-task:0x%x vgId:%d restored:%d status:%s not update checkpoint-info, checkpointId:%" PRId64 "->%"
// " failed", // PRId64
// pReq->taskId, vgId, restored, pStatus->name, pInfo->checkpointId, pReq->checkpointId); // " failed",
// taosThreadMutexUnlock(&pTask->lock); // pReq->taskId, vgId, restored, pStatus->name, pInfo->checkpointId, pReq->checkpointId);
// return TSDB_CODE_STREAM_TASK_IVLD_STATUS; // taosThreadMutexUnlock(&pTask->lock);
// } // return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
// }
if (!restored) { // during restore procedure, do update checkpoint-info if (!restored) { // during restore procedure, do update checkpoint-info
stDebug("s-task:%s vgId:%d status:%s update the checkpoint-info during restore, checkpointId:%" PRId64 "->%" PRId64 stDebug("s-task:%s vgId:%d status:%s update the checkpoint-info during restore, checkpointId:%" PRId64 "->%" PRId64
@ -568,12 +572,12 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l
} }
int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t dbRefId, ECHECKPOINT_BACKUP_TYPE type) { int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t dbRefId, ECHECKPOINT_BACKUP_TYPE type) {
char* path = NULL; char* path = NULL;
int32_t code = 0; int32_t code = 0;
SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES); SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES);
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
const char* idStr = pTask->id.idStr; const char* idStr = pTask->id.idStr;
if ((code = taskDbGenChkpUploadData(pTask->pBackend, pMeta->bkdChkptMgt, checkpointId, type, &path, toDelFiles, if ((code = taskDbGenChkpUploadData(pTask->pBackend, pMeta->bkdChkptMgt, checkpointId, type, &path, toDelFiles,
pTask->id.idStr)) != 0) { pTask->id.idStr)) != 0) {
@ -619,8 +623,8 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d
idStr, checkpointId, el, path); idStr, checkpointId, el, path);
taosRemoveDir(path); taosRemoveDir(path);
} else { } else {
stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " keep local checkpoint data, elapsed time:%.2fs", stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " keep local checkpoint data, elapsed time:%.2fs", idStr,
idStr, checkpointId, el); checkpointId, el);
} }
taosMemoryFree(path); taosMemoryFree(path);
@ -639,9 +643,10 @@ int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t checkpointI
} }
int64_t dbRefId = taskGetDBRef(pTask->pBackend); int64_t dbRefId = taskGetDBRef(pTask->pBackend);
void* pBackend = taskAcquireDb(dbRefId); void* pBackend = taskAcquireDb(dbRefId);
if (pBackend == NULL) { if (pBackend == NULL) {
stError("s-task:%s failed to acquire db during update checkpoint data, failed to upload checkpointData", pTask->id.idStr); stError("s-task:%s failed to acquire db during update checkpoint data, failed to upload checkpointData",
pTask->id.idStr);
return -1; return -1;
} }
@ -773,11 +778,11 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
ASSERT(pTask->info.taskLevel > TASK_LEVEL__SOURCE); ASSERT(pTask->info.taskLevel > TASK_LEVEL__SOURCE);
SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo)); SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo));
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pList, i); SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pList, i);
bool recved = false; bool recved = false;
for(int32_t j = 0; j < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++j) { for (int32_t j = 0; j < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++j) {
STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, j); STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, j);
if (pInfo->nodeId == pReady->upstreamNodeId) { if (pInfo->nodeId == pReady->upstreamNodeId) {
recved = true; recved = true;
@ -785,7 +790,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
} }
} }
if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message
streamTaskOpenUpstreamInput(pTask, pInfo->taskId); streamTaskOpenUpstreamInput(pTask, pInfo->taskId);
taosArrayPush(pNotSendList, pInfo); taosArrayPush(pNotSendList, pInfo);
} }
@ -870,7 +875,7 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId)
return false; return false;
} }
for(int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
STaskTriggerSendInfo* pSendInfo = taosArrayGet(pInfo->pDispatchTriggerList, i); STaskTriggerSendInfo* pSendInfo = taosArrayGet(pInfo->pDispatchTriggerList, i);
if (pSendInfo->nodeId != downstreamNodeId) { if (pSendInfo->nodeId != downstreamNodeId) {
continue; continue;
@ -939,10 +944,10 @@ int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) {
int32_t num = 0; int32_t num = 0;
taosThreadMutexLock(&pInfo->lock); taosThreadMutexLock(&pInfo->lock);
for(int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
if (p->recved) { if (p->recved) {
num ++; num++;
} }
} }
taosThreadMutexUnlock(&pInfo->lock); taosThreadMutexUnlock(&pInfo->lock);

59
source/libs/stream/src/streamDispatch.c
View File

@ -65,7 +65,7 @@ static int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTas
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void streamTaskSendRetrieveRsp(SStreamRetrieveReq *pReq, SRpcMsg* pRsp){ void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp) {
void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp)); void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp));
((SMsgHead*)buf)->vgId = htonl(pReq->srcNodeId); ((SMsgHead*)buf)->vgId = htonl(pReq->srcNodeId);
@ -119,7 +119,6 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
return code; return code;
} }
buf = NULL;
stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr,
pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId); pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId);
} }
@ -127,7 +126,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
return code; return code;
} }
static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBlock, SStreamRetrieveReq* req){ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBlock, SStreamRetrieveReq* req) {
SRetrieveTableRsp* pRetrieve = NULL; SRetrieveTableRsp* pRetrieve = NULL;
int32_t len = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; int32_t len = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN;
@ -147,7 +146,7 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl
pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey);
pRetrieve->version = htobe64(pBlock->info.version); pRetrieve->version = htobe64(pBlock->info.version);
int32_t actualLen = blockEncode(pBlock, pRetrieve->data+ PAYLOAD_PREFIX_LEN, numOfCols); int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, numOfCols);
SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen);
int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN;
@ -165,8 +164,8 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl
int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock) { int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock) {
SStreamRetrieveReq req; SStreamRetrieveReq req;
int32_t code = buildStreamRetrieveReq(pTask, pBlock, &req); int32_t code = buildStreamRetrieveReq(pTask, pBlock, &req);
if(code != 0){ if (code != 0) {
return code; return code;
} }
@ -310,7 +309,6 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD
addDispatchEntry(&pTask->msgInfo, pTask->outputInfo.fixedDispatcher.nodeId, now, true); addDispatchEntry(&pTask->msgInfo, pTask->outputInfo.fixedDispatcher.nodeId, now, true);
pTask->msgInfo.pData = pReqs; pTask->msgInfo.pData = pReqs;
} else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos;
int32_t numOfVgroups = taosArrayGetSize(vgInfo); int32_t numOfVgroups = taosArrayGetSize(vgInfo);
@ -568,8 +566,8 @@ void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) {
} }
} }
int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int64_t groupId,
int64_t groupId, int64_t now) { int64_t now) {
uint32_t hashValue = 0; uint32_t hashValue = 0;
SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos;
if (pTask->pNameMap == NULL) { if (pTask->pNameMap == NULL) {
@ -656,9 +654,7 @@ static void clearDispatchInfo(SDispatchMsgInfo* pInfo) {
pInfo->rspTs = -1; pInfo->rspTs = -1;
} }
static void updateDispatchInfo(SDispatchMsgInfo* pInfo, int64_t recvTs) { static void updateDispatchInfo(SDispatchMsgInfo* pInfo, int64_t recvTs) { pInfo->rspTs = recvTs; }
pInfo->rspTs = recvTs;
}
int32_t streamDispatchStreamBlock(SStreamTask* pTask) { int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
ASSERT((pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || ASSERT((pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH ||
@ -719,8 +715,8 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
taosThreadMutexLock(&pTask->msgInfo.lock); taosThreadMutexLock(&pTask->msgInfo.lock);
if (pTask->msgInfo.inMonitor == 0) { if (pTask->msgInfo.inMonitor == 0) {
int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS, ref, stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS,
tstrerror(code)); ref, tstrerror(code));
streamStartMonitorDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); streamStartMonitorDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS);
pTask->msgInfo.inMonitor = 1; pTask->msgInfo.inMonitor = 1;
} else { } else {
@ -829,13 +825,13 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
int32_t checkpointId = pActiveInfo->activeId; int32_t checkpointId = pActiveInfo->activeId;
int32_t notRsp = taosArrayGetSize(pNotRspList); int32_t notRsp = taosArrayGetSize(pNotRspList);
if (notRsp > 0) { // send checkpoint-ready msg again if (notRsp > 0) { // send checkpoint-ready msg again
for (int32_t i = 0; i < taosArrayGetSize(pNotRspList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pNotRspList); ++i) {
int32_t taskId = *(int32_t*)taosArrayGet(pNotRspList, i); int32_t taskId = *(int32_t*)taosArrayGet(pNotRspList, i);
for (int32_t j = 0; j < num; ++j) { for (int32_t j = 0; j < num; ++j) {
STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pList, j); STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pList, j);
if (taskId == pReadyInfo->upstreamTaskId) { // send msg again if (taskId == pReadyInfo->upstreamTaskId) { // send msg again
SRpcMsg msg = {0}; SRpcMsg msg = {0};
initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId, initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId,
@ -880,7 +876,8 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i); STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i);
SRpcMsg msg = {0}; SRpcMsg msg = {0};
initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId, &msg); initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId,
&msg);
tmsgSendReq(&pInfo->upstreamNodeEpset, &msg); tmsgSendReq(&pInfo->upstreamNodeEpset, &msg);
stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel, stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel,
@ -918,9 +915,11 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
tmsgSendRsp(&pInfo->msg); tmsgSendRsp(&pInfo->msg);
taosArrayClear(pList); taosArrayClear(pList);
stDebug("s-task:%s level:%d checkpoint-source rsp completed msg sent to mnode", pTask->id.idStr, pTask->info.taskLevel); stDebug("s-task:%s level:%d checkpoint-source rsp completed msg sent to mnode", pTask->id.idStr,
pTask->info.taskLevel);
} else { } else {
stDebug("s-task:%s level:%d already send checkpoint-source rsp success to mnode", pTask->id.idStr, pTask->info.taskLevel); stDebug("s-task:%s level:%d already send checkpoint-source rsp success to mnode", pTask->id.idStr,
pTask->info.taskLevel);
} }
taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
@ -1010,7 +1009,7 @@ FAIL:
} }
int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg, int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg,
int32_t setCode) { int32_t setCode) {
int32_t len = 0; int32_t len = 0;
int32_t code = 0; int32_t code = 0;
SEncoder encoder; SEncoder encoder;
@ -1049,7 +1048,8 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
return 0; return 0;
} }
int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask) { int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo,
SStreamTask* pTask) {
STaskCheckpointReadyInfo info = { STaskCheckpointReadyInfo info = {
.recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId}; .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId};
@ -1070,7 +1070,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
stError("s-task:%s checkpointId:%" PRId64 " transId:%d not completed, new transId:%d checkpointId:%" PRId64 stError("s-task:%s checkpointId:%" PRId64 " transId:%d not completed, new transId:%d checkpointId:%" PRId64
" recv from mnode", " recv from mnode",
pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId); pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId);
ASSERT(0); // failed to handle it ASSERT(0); // failed to handle it
} }
} else { } else {
taosArrayPush(pActiveInfo->pReadyMsgList, &info); taosArrayPush(pActiveInfo->pReadyMsgList, &info);
@ -1105,8 +1105,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
STaskCheckpointReadyInfo info = {0}; STaskCheckpointReadyInfo info = {0};
initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId); initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId);
stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64 stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64 "-0x%x (vgId:%d) idx:%d",
"-0x%x (vgId:%d) idx:%d",
pTask->id.idStr, pTask->info.taskLevel, pTask->id.streamId, pInfo->taskId, pInfo->nodeId, index); pTask->id.idStr, pTask->info.taskLevel, pTask->id.streamId, pInfo->taskId, pInfo->nodeId, index);
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
@ -1178,7 +1177,7 @@ static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int3
bool updated = false; bool updated = false;
taosThreadMutexLock(&pMsgInfo->lock); taosThreadMutexLock(&pMsgInfo->lock);
for(int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) {
SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j);
if (pEntry->nodeId == vgId) { if (pEntry->nodeId == vgId) {
ASSERT(!alreadySet); ASSERT(!alreadySet);
@ -1186,7 +1185,7 @@ static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int3
pEntry->status = code; pEntry->status = code;
alreadySet = true; alreadySet = true;
updated = true; updated = true;
stDebug("s-task:%s record the rsp recv, ts:%"PRId64" code:%d, idx:%d", id, now, code, j); stDebug("s-task:%s record the rsp recv, ts:%" PRId64 " code:%d, idx:%d", id, now, code, j);
} }
if (pEntry->rspTs != -1) { if (pEntry->rspTs != -1) {
@ -1301,12 +1300,14 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
if (notRsp > 0) { if (notRsp > 0) {
stDebug( stDebug(
"s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, waiting " "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, "
"waiting "
"for %d rsp", "for %d rsp",
id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), notRsp); id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), notRsp);
} else { } else {
stDebug( stDebug(
"s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, all rsp", "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, all "
"rsp",
id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code)); id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code));
} }
} else { } else {
@ -1407,7 +1408,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
if (pReq->stage > pInfo->stage) { if (pReq->stage > pInfo->stage) {
// upstream task has restarted/leader-follower switch/transferred to other dnodes // upstream task has restarted/leader-follower switch/transferred to other dnodes
stError("s-task:%s upstream task:0x%x (vgId:%d) has restart/leader-switch/vnode-transfer, prev stage:%" PRId64 stError("s-task:%s upstream task:0x%x (vgId:%d) has restart/leader-switch/vnode-transfer, prev stage:%" PRId64
", current:%" PRId64 " dispatch msg rejected", ", current:%" PRId64 " dispatch msg rejected",
id, pReq->upstreamTaskId, pReq->upstreamNodeId, pInfo->stage, pReq->stage); id, pReq->upstreamTaskId, pReq->upstreamNodeId, pInfo->stage, pReq->stage);
status = TASK_INPUT_STATUS__REFUSED; status = TASK_INPUT_STATUS__REFUSED;
} else { } else {

25
source/libs/stream/src/streamMeta.c
View File

@ -151,7 +151,7 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) {
int8_t ret = STREAM_STATA_COMPATIBLE; int8_t ret = STREAM_STATA_COMPATIBLE;
TBC* pCur = NULL; TBC* pCur = NULL;
if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream
return ret; return ret;
} }
@ -557,7 +557,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
return -1; return -1;
} }
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) {
pTask->ver = SSTREAM_TASK_VER; pTask->ver = SSTREAM_TASK_VER;
} }
@ -907,7 +907,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
if (p == NULL) { if (p == NULL) {
code = pMeta->buildTaskFn(pMeta->ahandle, pTask, pTask->chkInfo.checkpointVer + 1); code = pMeta->buildTaskFn(pMeta->ahandle, pTask, pTask->chkInfo.checkpointVer + 1);
if (code < 0) { if (code < 0) {
stError("failed to expand s-task:0x%"PRIx64", code:%s, continue", id.taskId, tstrerror(terrno)); stError("failed to expand s-task:0x%" PRIx64 ", code:%s, continue", id.taskId, tstrerror(terrno));
tFreeStreamTask(pTask); tFreeStreamTask(pTask);
continue; continue;
} }
@ -1012,7 +1012,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) {
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i);
STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId};
SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (pTask == NULL) { if (pTask == NULL) {
continue; continue;
@ -1052,12 +1052,14 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) {
} }
if ((*pTask)->chkInfo.pActiveInfo->activeId != 0) { if ((*pTask)->chkInfo.pActiveInfo->activeId != 0) {
entry.checkpointInfo.failed = ((*pTask)->chkInfo.pActiveInfo->failedId >= (*pTask)->chkInfo.pActiveInfo->activeId) ? 1 : 0; entry.checkpointInfo.failed =
((*pTask)->chkInfo.pActiveInfo->failedId >= (*pTask)->chkInfo.pActiveInfo->activeId) ? 1 : 0;
entry.checkpointInfo.activeId = (*pTask)->chkInfo.pActiveInfo->activeId; entry.checkpointInfo.activeId = (*pTask)->chkInfo.pActiveInfo->activeId;
entry.checkpointInfo.activeTransId = (*pTask)->chkInfo.pActiveInfo->transId; entry.checkpointInfo.activeTransId = (*pTask)->chkInfo.pActiveInfo->transId;
if (entry.checkpointInfo.failed) { if (entry.checkpointInfo.failed) {
stInfo("s-task:%s set kill checkpoint trans in hb, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.pActiveInfo->transId); stInfo("s-task:%s set kill checkpoint trans in hb, transId:%d", (*pTask)->id.idStr,
(*pTask)->chkInfo.pActiveInfo->transId);
} }
} }
@ -1384,7 +1386,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expa
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
stInfo("vgId:%d start to check all %d stream task(s) downstream status, start ts:%"PRId64, vgId, numOfTasks, now); stInfo("vgId:%d start to check all %d stream task(s) downstream status, start ts:%" PRId64, vgId, numOfTasks, now);
if (numOfTasks == 0) { if (numOfTasks == 0) {
stInfo("vgId:%d no tasks to be started", pMeta->vgId); stInfo("vgId:%d no tasks to be started", pMeta->vgId);
@ -1513,8 +1515,8 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta) {
int32_t num = taosArrayGetSize(pMeta->pTaskList); int32_t num = taosArrayGetSize(pMeta->pTaskList);
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i);
STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId};
SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask == NULL) { if (ppTask == NULL) {
continue; continue;
} }
@ -1598,7 +1600,7 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
if (pStartInfo->startAllTasks != 1) { if (pStartInfo->startAllTasks != 1) {
int64_t el = endTs - startTs; int64_t el = endTs - startTs;
stDebug("vgId:%d not start all task(s), not record status, s-task:0x%x launch succ:%d elapsed time:%" PRId64 "ms", stDebug("vgId:%d not start all task(s), not record status, s-task:0x%x launch succ:%d elapsed time:%" PRId64 "ms",
pMeta->vgId, taskId, ready, el); pMeta->vgId, taskId, ready, el);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
return 0; return 0;
} }
@ -1725,7 +1727,8 @@ void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SSt
taosHashPut(pMeta->updateInfo.pTasks, &hEntry, sizeof(hEntry), NULL, 0); taosHashPut(pMeta->updateInfo.pTasks, &hEntry, sizeof(hEntry), NULL, 0);
stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask/hTask closed, elapsed:%" PRId64 stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask/hTask closed, elapsed:%" PRId64
" ms", id, vgId, transId, el); " ms",
id, vgId, transId, el);
} else { } else {
stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms", stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms",
id, vgId, transId, el); id, vgId, transId, el);

5
source/libs/stream/src/streamSched.c
View File

@ -26,7 +26,8 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask) {
stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.delaySchedParam); stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.delaySchedParam);
pTask->schedInfo.pDelayTimer = taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.delaySchedParam, pTask, streamTimer); pTask->schedInfo.pDelayTimer =
taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.delaySchedParam, pTask, streamTimer);
pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE; pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE;
} }
@ -78,7 +79,7 @@ int32_t streamTaskResumeInFuture(SStreamTask* pTask) {
pTask->status.schedIdleTime, ref); pTask->status.schedIdleTime, ref);
// add one ref count for task // add one ref count for task
/*SStreamTask* pAddRefTask = */streamMetaAcquireOneTask(pTask); /*SStreamTask* pAddRefTask = */ streamMetaAcquireOneTask(pTask);
if (pTask->schedInfo.pIdleTimer == NULL) { if (pTask->schedInfo.pIdleTimer == NULL) {
pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer); pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer);

22
source/libs/stream/src/streamStartHistory.c
View File

@ -105,9 +105,11 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration)
stDebug("s-task:%s scan-history resumed in %.2fs, ref:%d", pTask->id.idStr, numOfTicks * 0.1, ref); stDebug("s-task:%s scan-history resumed in %.2fs, ref:%d", pTask->id.idStr, numOfTicks * 0.1, ref);
if (pTask->schedHistoryInfo.pTimer == NULL) { if (pTask->schedHistoryInfo.pTimer == NULL) {
pTask->schedHistoryInfo.pTimer = taosTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer); pTask->schedHistoryInfo.pTimer =
taosTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer);
} else { } else {
taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, &pTask->schedHistoryInfo.pTimer); taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer,
&pTask->schedHistoryInfo.pTimer);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -204,7 +206,8 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
// check stream task status in the first place. // check stream task status in the first place.
SStreamTaskState* pStatus = streamTaskGetStatus(pTask); SStreamTaskState* pStatus = streamTaskGetStatus(pTask);
if (pStatus->state != TASK_STATUS__READY && pStatus->state != TASK_STATUS__HALT && pStatus->state != TASK_STATUS__PAUSE) { if (pStatus->state != TASK_STATUS__READY && pStatus->state != TASK_STATUS__HALT &&
pStatus->state != TASK_STATUS__PAUSE) {
stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId, stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId,
pStatus->name); pStatus->name);
@ -389,7 +392,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
// not in timer anymore // not in timer anymore
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:0x%x fill-history task launch completed, retry times:%d, ref:%d", (int32_t)pInfo->id.taskId, stDebug("s-task:0x%x fill-history task launch completed, retry times:%d, ref:%d", (int32_t)pInfo->id.taskId,
pHTaskInfo->retryTimes, ref); pHTaskInfo->retryTimes, ref);
} }
streamMetaReleaseTask(pMeta, pHTask); streamMetaReleaseTask(pMeta, pHTask);
} }
@ -434,7 +437,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
stWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since not built yet", idStr, pMeta->vgId, hTaskId); stWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since not built yet", idStr, pMeta->vgId, hTaskId);
STaskId id = streamTaskGetTaskId(pTask); STaskId id = streamTaskGetTaskId(pTask);
SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId); SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId);
if (pInfo == NULL) { if (pInfo == NULL) {
stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr); stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr);
@ -494,8 +497,8 @@ bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t nextProcessVe
// [pTask->dataRange.range.maxVer, ver1] // [pTask->dataRange.range.maxVer, ver1]
pTask->step2Range.minVer = walScanStartVer; pTask->step2Range.minVer = walScanStartVer;
pTask->step2Range.maxVer = nextProcessVer - 1; pTask->step2Range.maxVer = nextProcessVer - 1;
stDebug("s-task:%s set step2 verRange:%" PRId64 "-%" PRId64 ", step1 verRange:%" PRId64 "-%" PRId64, pTask->id.idStr, stDebug("s-task:%s set step2 verRange:%" PRId64 "-%" PRId64 ", step1 verRange:%" PRId64 "-%" PRId64,
pTask->step2Range.minVer, pTask->step2Range.maxVer, pRange->minVer, pRange->maxVer); pTask->id.idStr, pTask->step2Range.minVer, pTask->step2Range.maxVer, pRange->minVer, pRange->maxVer);
return false; return false;
} }
} }
@ -525,7 +528,7 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
pRange->range.maxVer); pRange->range.maxVer);
SVersionRange verRange = pRange->range; SVersionRange verRange = pRange->range;
STimeWindow win = pRange->window; STimeWindow win = pRange->window;
streamSetParamForStreamScannerStep2(pTask, &verRange, &win); streamSetParamForStreamScannerStep2(pTask, &verRange, &win);
} }
} }
@ -553,7 +556,8 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) {
// release the task. // release the task.
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
} else { } else {
taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, &pTask->schedHistoryInfo.pTimer); taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer,
&pTask->schedHistoryInfo.pTimer);
} }
} }

37
source/libs/stream/src/streamTask.c
View File

@ -121,7 +121,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset,
pTask->id.idStr = taosStrdup(buf); pTask->id.idStr = taosStrdup(buf);
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
pTask->status.taskStatus = fillHistory? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY; pTask->status.taskStatus = fillHistory ? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY;
pTask->inputq.status = TASK_INPUT_STATUS__NORMAL; pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL; pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;
@ -208,9 +208,8 @@ void tFreeStreamTask(SStreamTask* pTask) {
stDebug("s-task:0x%x task exec summary: create:%" PRId64 ", init:%" PRId64 ", start:%" PRId64 stDebug("s-task:0x%x task exec summary: create:%" PRId64 ", init:%" PRId64 ", start:%" PRId64
", updateCount:%d latestUpdate:%" PRId64 ", latestCheckPoint:%" PRId64 ", ver:%" PRId64 ", updateCount:%d latestUpdate:%" PRId64 ", latestCheckPoint:%" PRId64 ", ver:%" PRId64
" nextProcessVer:%" PRId64 ", checkpointCount:%d", " nextProcessVer:%" PRId64 ", checkpointCount:%d",
taskId, pStatis->created, pStatis->checkTs, pStatis->readyTs, pStatis->updateCount, taskId, pStatis->created, pStatis->checkTs, pStatis->readyTs, pStatis->updateCount, pStatis->latestUpdateTs,
pStatis->latestUpdateTs, pCkInfo->checkpointId, pCkInfo->checkpointVer, pCkInfo->nextProcessVer, pCkInfo->checkpointId, pCkInfo->checkpointVer, pCkInfo->nextProcessVer, pStatis->checkpoint);
pStatis->checkpoint);
// remove the ref by timer // remove the ref by timer
while (pTask->status.timerActive > 0) { while (pTask->status.timerActive > 0) {
@ -224,12 +223,12 @@ void tFreeStreamTask(SStreamTask* pTask) {
} }
if (pTask->hTaskInfo.pTimer != NULL) { if (pTask->hTaskInfo.pTimer != NULL) {
/*bool ret = */taosTmrStop(pTask->hTaskInfo.pTimer); /*bool ret = */ taosTmrStop(pTask->hTaskInfo.pTimer);
pTask->hTaskInfo.pTimer = NULL; pTask->hTaskInfo.pTimer = NULL;
} }
if (pTask->msgInfo.pRetryTmr != NULL) { if (pTask->msgInfo.pRetryTmr != NULL) {
/*bool ret = */taosTmrStop(pTask->msgInfo.pRetryTmr); /*bool ret = */ taosTmrStop(pTask->msgInfo.pRetryTmr);
pTask->msgInfo.pRetryTmr = NULL; pTask->msgInfo.pRetryTmr = NULL;
} }
@ -414,7 +413,8 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
streamTaskInitTokenBucket(pOutputInfo->pTokenBucket, 35, 35, tsSinkDataRate, pTask->id.idStr); streamTaskInitTokenBucket(pOutputInfo->pTokenBucket, 35, 35, tsSinkDataRate, pTask->id.idStr);
pOutputInfo->pNodeEpsetUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset)); pOutputInfo->pNodeEpsetUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset));
if (pOutputInfo->pNodeEpsetUpdateList == NULL) { if (pOutputInfo->pNodeEpsetUpdateList == NULL) {
stError("s-task:%s failed to prepare downstreamUpdateList, code:%s", pTask->id.idStr, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); stError("s-task:%s failed to prepare downstreamUpdateList, code:%s", pTask->id.idStr,
tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -448,9 +448,7 @@ int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask) {
} }
} }
int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask) { int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask) { return taosArrayGetSize(pTask->upstreamInfo.pList); }
return taosArrayGetSize(pTask->upstreamInfo.pList);
}
int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask) { int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask) {
SStreamUpstreamEpInfo* pEpInfo = createStreamTaskEpInfo(pUpstreamTask); SStreamUpstreamEpInfo* pEpInfo = createStreamTaskEpInfo(pUpstreamTask);
@ -512,7 +510,7 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo
pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH; pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH;
} }
void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool *pUpdated) { void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated) {
char buf[512] = {0}; char buf[512] = {0};
epsetToStr(pEpSet, buf, tListLen(buf)); epsetToStr(pEpSet, buf, tListLen(buf));
@ -740,10 +738,10 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI
} }
int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask) { int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask) {
int32_t code; int32_t code;
int32_t tlen = 0; int32_t tlen = 0;
int32_t vgId = pTask->pMeta->vgId; int32_t vgId = pTask->pMeta->vgId;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
SActiveCheckpointInfo* pActive = pCheckpointInfo->pActiveInfo; SActiveCheckpointInfo* pActive = pCheckpointInfo->pActiveInfo;
SCheckpointReport req = {.streamId = pTask->id.streamId, SCheckpointReport req = {.streamId = pTask->id.streamId,
@ -862,7 +860,8 @@ void streamTaskResume(SStreamTask* pTask) {
int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num);
} else { } else {
stInfo("s-task:%s status:%s no need to resume, paused task(s):%d", pTask->id.idStr, prevState.name, pMeta->numOfPausedTasks); stInfo("s-task:%s status:%s no need to resume, paused task(s):%d", pTask->id.idStr, prevState.name,
pMeta->numOfPausedTasks);
} }
} }
@ -925,7 +924,7 @@ SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) {
} }
} else if (pTask->info.taskLevel == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pTask->info.taskLevel == TASK_OUTPUT__SHUFFLE_DISPATCH) {
SArray* pList = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; SArray* pList = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos;
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
SVgroupInfo* pVgInfo = taosArrayGet(pList, i); SVgroupInfo* pVgInfo = taosArrayGet(pList, i);
if (pVgInfo->taskId == taskId) { if (pVgInfo->taskId == taskId) {
return &pVgInfo->epSet; return &pVgInfo->epSet;
@ -979,9 +978,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) {
return streamTrySchedExec(pTask); return streamTrySchedExec(pTask);
} }
void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) { void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) { pTask->status.removeBackendFiles = true; }
pTask->status.removeBackendFiles = true;
}
int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId) { int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId) {
if (pTransId != NULL) { if (pTransId != NULL) {

4
source/libs/stream/src/streamTaskSm.c
View File

@ -200,7 +200,6 @@ static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent eve
SStreamTaskSM* pSM = pTask->status.pSM; SStreamTaskSM* pSM = pTask->status.pSM;
bool removed = false; bool removed = false;
taosThreadMutexLock(&pTask->lock);
int32_t num = taosArrayGetSize(pSM->pWaitingEventList); int32_t num = taosArrayGetSize(pSM->pWaitingEventList);
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
@ -218,7 +217,6 @@ static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent eve
stDebug("s-task:%s failed to remove event:%s in waiting list", pTask->id.idStr, StreamTaskEventList[event].name); stDebug("s-task:%s failed to remove event:%s in waiting list", pTask->id.idStr, StreamTaskEventList[event].name);
} }
taosThreadMutexUnlock(&pTask->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -671,4 +669,4 @@ void doInitStateTransferTable(void) {
trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); taosArrayPush(streamTaskSMTrans, &trans);
} }
//clang-format on //clang-format on

7
source/libs/stream/src/streammsg.c
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h"
#include "streammsg.h" #include "streammsg.h"
#include "os.h"
#include "tstream.h" #include "tstream.h"
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) { int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) {
@ -77,7 +77,6 @@ int32_t tEncodeStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckp
return pEncoder->pos; return pEncoder->pos;
} }
int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpdateMsg* pMsg) { int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpdateMsg* pMsg) {
if (tStartEncode(pEncoder) < 0) return -1; if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pMsg->streamId) < 0) return -1; if (tEncodeI64(pEncoder, pMsg->streamId) < 0) return -1;
@ -99,7 +98,6 @@ int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpda
return pEncoder->pos; return pEncoder->pos;
} }
int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg) { int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg) {
if (tStartDecode(pDecoder) < 0) return -1; if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pMsg->streamId) < 0) return -1; if (tDecodeI64(pDecoder, &pMsg->streamId) < 0) return -1;
@ -180,7 +178,6 @@ int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp)
return 0; return 0;
} }
int32_t tEncodeStreamCheckpointReadyMsg(SEncoder* pEncoder, const SStreamCheckpointReadyMsg* pReq) { int32_t tEncodeStreamCheckpointReadyMsg(SEncoder* pEncoder, const SStreamCheckpointReadyMsg* pReq) {
if (tStartEncode(pEncoder) < 0) return -1; if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
@ -589,7 +586,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
if (tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1;
} }
if (tDecodeI64(pDecoder, &pTask->info.delaySchedParam) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->info.delaySchedParam) < 0) return -1;
if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER){ if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) {
if (tDecodeI8(pDecoder, &pTask->subtableWithoutMd5) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->subtableWithoutMd5) < 0) return -1;
} }
if (tDecodeCStrTo(pDecoder, pTask->reserve) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->reserve) < 0) return -1;

20
source/libs/tdb/src/db/tdbBtree.c
View File

@ -351,7 +351,12 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
} }
pCell = tdbPageGetCell(btc.pPage, btc.idx); pCell = tdbPageGetCell(btc.pPage, btc.idx);
tdbBtreeDecodeCell(btc.pPage, pCell, &cd, btc.pTxn, pBt); ret = tdbBtreeDecodeCell(btc.pPage, pCell, &cd, btc.pTxn, pBt);
if (ret < 0) {
tdbBtcClose(&btc);
tdbError("tdb/btree-pget: decode cell failed with ret: %d.", ret);
return -1;
}
if (ppKey) { if (ppKey) {
pTKey = tdbRealloc(*ppKey, cd.kLen); pTKey = tdbRealloc(*ppKey, cd.kLen);
@ -1072,6 +1077,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
// pack partial val to local if any space left // pack partial val to local if any space left
if (nLocal > nHeader + kLen + sizeof(SPgno)) { if (nLocal > nHeader + kLen + sizeof(SPgno)) {
if (ASSERT(pVal != NULL && vLen != 0)) { if (ASSERT(pVal != NULL && vLen != 0)) {
tdbFree(pBuf);
return -1; return -1;
} }
memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno)); memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno));
@ -1859,7 +1865,11 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx);
tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd, pBtc->pTxn, pBtc->pBt); ret = tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd, pBtc->pTxn, pBtc->pBt);
if (ret < 0) {
tdbError("tdb/btree-next: decode cell failed with ret: %d.", ret);
return -1;
}
pKey = tdbRealloc(*ppKey, cd.kLen); pKey = tdbRealloc(*ppKey, cd.kLen);
if (pKey == NULL) { if (pKey == NULL) {
@ -1918,7 +1928,11 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx);
tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd, pBtc->pTxn, pBtc->pBt); ret = tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd, pBtc->pTxn, pBtc->pBt);
if (ret < 0) {
tdbError("tdb/btree-prev: decode cell failed with ret: %d.", ret);
return -1;
}
pKey = tdbRealloc(*ppKey, cd.kLen); pKey = tdbRealloc(*ppKey, cd.kLen);
if (pKey == NULL) { if (pKey == NULL) {

29
source/libs/transport/src/transCli.c
View File

@ -672,6 +672,7 @@ static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) {
SCliConn* conn = QUEUE_DATA(h, SCliConn, q); SCliConn* conn = QUEUE_DATA(h, SCliConn, q);
conn->status = ConnNormal; conn->status = ConnNormal;
QUEUE_INIT(&conn->q); QUEUE_INIT(&conn->q);
tDebug("conn %p get from pool, pool size: %d, dst: %s", conn, conn->list->size, conn->dstAddr);
if (conn->task != NULL) { if (conn->task != NULL) {
transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task);
@ -754,6 +755,7 @@ static SCliConn* getConnFromPool2(SCliThrd* pThrd, char* key, SCliMsg** pMsg) {
SCliConn* conn = QUEUE_DATA(h, SCliConn, q); SCliConn* conn = QUEUE_DATA(h, SCliConn, q);
conn->status = ConnNormal; conn->status = ConnNormal;
QUEUE_INIT(&conn->q); QUEUE_INIT(&conn->q);
tDebug("conn %p get from pool, pool size: %d, dst: %s", conn, conn->list->size, conn->dstAddr);
if (conn->task != NULL) { if (conn->task != NULL) {
transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task);
@ -806,6 +808,7 @@ static void addConnToPool(void* pool, SCliConn* conn) {
conn->status = ConnInPool; conn->status = ConnInPool;
QUEUE_PUSH(&conn->list->conns, &conn->q); QUEUE_PUSH(&conn->list->conns, &conn->q);
conn->list->size += 1; conn->list->size += 1;
tDebug("conn %p added to pool, pool size: %d, dst: %s", conn, conn->list->size, conn->dstAddr);
if (conn->list->size >= 10) { if (conn->list->size >= 10) {
STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg));
@ -825,6 +828,9 @@ static int32_t allocConnRef(SCliConn* conn, bool update) {
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
exh->handle = conn; exh->handle = conn;
exh->pThrd = conn->hostThrd; exh->pThrd = conn->hostThrd;
QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch);
exh->refId = transAddExHandle(transGetRefMgt(), exh); exh->refId = transAddExHandle(transGetRefMgt(), exh);
QUEUE_INIT(&exh->q); QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch); taosInitRWLatch(&exh->latch);
@ -849,6 +855,8 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) {
taosWLockLatch(&exh->latch); taosWLockLatch(&exh->latch);
exh->handle = conn; exh->handle = conn;
exh->pThrd = conn->hostThrd; exh->pThrd = conn->hostThrd;
taosWUnLockLatch(&exh->latch);
conn->refId = exh->refId; conn->refId = exh->refId;
taosWUnLockLatch(&exh->latch); taosWUnLockLatch(&exh->latch);
@ -911,13 +919,11 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) {
uv_timer_init(pThrd->loop, timer); uv_timer_init(pThrd->loop, timer);
} }
timer->data = conn; timer->data = conn;
conn->timer = timer;
conn->timer = timer;
conn->connReq.data = conn; conn->connReq.data = conn;
transReqQueueInit(&conn->wreqQueue); transReqQueueInit(&conn->wreqQueue);
transQueueInit(&conn->cliMsgs, NULL); transQueueInit(&conn->cliMsgs, NULL);
transInitBuffer(&conn->readBuf); transInitBuffer(&conn->readBuf);
QUEUE_INIT(&conn->q); QUEUE_INIT(&conn->q);
conn->hostThrd = pThrd; conn->hostThrd = pThrd;
@ -1027,7 +1033,7 @@ static void cliSendCb(uv_write_t* req, int status) {
SCliConn* pConn = transReqQueueRemove(req); SCliConn* pConn = transReqQueueRemove(req);
if (pConn == NULL) return; if (pConn == NULL) return;
SCliMsg* pMsg = !transQueueEmpty(&pConn->cliMsgs) ? transQueueGet(&pConn->cliMsgs, 0) : NULL; SCliMsg* pMsg = transQueueGet(&pConn->cliMsgs, 0);
if (pMsg != NULL) { if (pMsg != NULL) {
int64_t cost = taosGetTimestampUs() - pMsg->st; int64_t cost = taosGetTimestampUs() - pMsg->st;
if (cost > 1000 * 50) { if (cost > 1000 * 50) {
@ -1133,6 +1139,7 @@ void cliSend(SCliConn* pConn) {
STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg);
if (pMsg->pCont == 0) { if (pMsg->pCont == 0) {
pMsg->pCont = (void*)rpcMallocCont(0); pMsg->pCont = (void*)rpcMallocCont(0);
tDebug("malloc memory: %p", pMsg->pCont);
pMsg->contLen = 0; pMsg->contLen = 0;
} }
@ -1579,7 +1586,7 @@ static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn) {
tinet_ntoa(old, *v); tinet_ntoa(old, *v);
tinet_ntoa(new, addr); tinet_ntoa(new, addr);
tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new);
taosHashPut(cache, fqdn, strlen(fqdn) + 1, &addr, sizeof(addr)); taosHashPut(cache, fqdn, strlen(fqdn), &addr, sizeof(addr));
} }
} }
return; return;
@ -1914,7 +1921,7 @@ void cliDestroyConnMsgs(SCliConn* conn, bool destroy) {
} }
} }
void cliIteraConnMsgs(SCliConn* conn) { void cliConnFreeMsgs(SCliConn* conn) {
SCliThrd* pThrd = conn->hostThrd; SCliThrd* pThrd = conn->hostThrd;
STrans* pTransInst = pThrd->pTransInst; STrans* pTransInst = pThrd->pTransInst;
@ -1956,7 +1963,7 @@ bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead) {
} }
} }
cliIteraConnMsgs(conn); cliConnFreeMsgs(conn);
tDebug("%s conn %p receive release request, refId:%" PRId64 "", CONN_GET_INST_LABEL(conn), conn, conn->refId); tDebug("%s conn %p receive release request, refId:%" PRId64 "", CONN_GET_INST_LABEL(conn), conn, conn->refId);
destroyCmsg(pMsg); destroyCmsg(pMsg);
@ -2017,6 +2024,7 @@ static FORCE_INLINE void destroyCmsg(void* arg) {
if (pMsg == NULL) { if (pMsg == NULL) {
return; return;
} }
tDebug("free memory:%p, free ctx: %p", pMsg, pMsg->ctx);
transDestroyConnCtx(pMsg->ctx); transDestroyConnCtx(pMsg->ctx);
transFreeMsg(pMsg->msg.pCont); transFreeMsg(pMsg->msg.pCont);
@ -2523,6 +2531,8 @@ static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(STrans* trans, int64_t
SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle); SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle);
if (exh == NULL) { if (exh == NULL) {
return NULL; return NULL;
} else {
tDebug("conn %p got", exh->handle);
} }
taosWLockLatch(&exh->latch); taosWLockLatch(&exh->latch);
if (exh->pThrd == NULL && trans != NULL) { if (exh->pThrd == NULL && trans != NULL) {
@ -2568,7 +2578,7 @@ int transReleaseCliHandle(void* handle) {
cmsg->ctx = pCtx; cmsg->ctx = pCtx;
STraceId* trace = &tmsg.info.traceId; STraceId* trace = &tmsg.info.traceId;
tGDebug("send release request at thread:%08" PRId64 "", pThrd->pid); tGDebug("send release request at thread:%08" PRId64 ", malloc memory:%p", pThrd->pid, cmsg);
if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) { if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) {
destroyCmsg(cmsg); destroyCmsg(cmsg);
@ -2819,6 +2829,9 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) {
int64_t transAllocHandle() { int64_t transAllocHandle() {
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch);
exh->refId = transAddExHandle(transGetRefMgt(), exh); exh->refId = transAddExHandle(transGetRefMgt(), exh);
QUEUE_INIT(&exh->q); QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch); taosInitRWLatch(&exh->latch);

6
source/libs/transport/src/transComm.c
View File

@ -305,7 +305,7 @@ void transCtxCleanup(STransCtx* ctx) {
ctx->freeFunc(iter->val); ctx->freeFunc(iter->val);
iter = taosHashIterate(ctx->args, iter); iter = taosHashIterate(ctx->args, iter);
} }
ctx->freeFunc(ctx->brokenVal.val); if (ctx->freeFunc) ctx->freeFunc(ctx->brokenVal.val);
taosHashCleanup(ctx->args); taosHashCleanup(ctx->args);
ctx->args = NULL; ctx->args = NULL;
} }
@ -676,6 +676,10 @@ void transDestroyExHandle(void* handle) {
if (handle == NULL) { if (handle == NULL) {
return; return;
} }
SExHandle* eh = handle;
if (!QUEUE_IS_EMPTY(&eh->q)) {
tDebug("handle %p mem leak", handle);
}
taosMemoryFree(handle); taosMemoryFree(handle);
} }

1
source/libs/transport/src/transSvr.c
View File

@ -1095,6 +1095,7 @@ static FORCE_INLINE SSvrConn* createConn(void* hThrd) {
exh->handle = pConn; exh->handle = pConn;
exh->pThrd = pThrd; exh->pThrd = pThrd;
exh->refId = transAddExHandle(transGetRefMgt(), exh); exh->refId = transAddExHandle(transGetRefMgt(), exh);
QUEUE_INIT(&exh->q);
transAcquireExHandle(transGetRefMgt(), exh->refId); transAcquireExHandle(transGetRefMgt(), exh->refId);
STrans* pTransInst = pThrd->pTransInst; STrans* pTransInst = pThrd->pTransInst;

160
source/util/src/tdecompress.c
View File

@ -14,8 +14,8 @@
*/ */
#include "os.h" #include "os.h"
#include "ttypes.h"
#include "tcompression.h" #include "tcompression.h"
#include "ttypes.h"
int32_t getWordLength(char type) { int32_t getWordLength(char type) {
int32_t wordLength = 0; int32_t wordLength = 0;
@ -54,7 +54,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements,
#if __AVX2__ || __AVX512F__ #if __AVX2__ || __AVX512F__
while (_pos < nelements) { while (_pos < nelements) {
uint64_t w = *(uint64_t*) ip; uint64_t w = *(uint64_t *)ip;
char selector = (char)(w & INT64MASK(4)); // selector = 4 char selector = (char)(w & INT64MASK(4)); // selector = 4
char bit = bit_per_integer[(int32_t)selector]; // bit = 3 char bit = bit_per_integer[(int32_t)selector]; // bit = 3
@ -67,22 +67,22 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements,
switch (type) { switch (type) {
case TSDB_DATA_TYPE_BIGINT: { case TSDB_DATA_TYPE_BIGINT: {
int64_t* p = (int64_t*) output; int64_t *p = (int64_t *)output;
int32_t gRemainder = (nelements - _pos); int32_t gRemainder = (nelements - _pos);
int32_t num = (gRemainder > elems)? elems:gRemainder; int32_t num = (gRemainder > elems) ? elems : gRemainder;
int32_t batch = 0; int32_t batch = 0;
int32_t remain = 0; int32_t remain = 0;
if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) {
#if __AVX512F__ #if __AVX512F__
batch = num >> 3; batch = num >> 3;
remain = num & 0x07; remain = num & 0x07;
#endif #endif
} else if (tsSIMDEnable && tsAVX2Supported) { } else if (tsSIMDEnable && tsAVX2Supported) {
#if __AVX2__ #if __AVX2__
batch = num >> 2; batch = num >> 2;
remain = num & 0x03; remain = num & 0x03;
#endif #endif
} }
@ -90,12 +90,12 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements,
if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) {
#if __AVX512F__ #if __AVX512F__
for (int32_t i = 0; i < batch; ++i) { for (int32_t i = 0; i < batch; ++i) {
__m512i prev = _mm512_set1_epi64(prevValue); __m512i prev = _mm512_set1_epi64(prevValue);
_mm512_storeu_si512((__m512i *)&p[_pos], prev); _mm512_storeu_si512((__m512i *)&p[_pos], prev);
_pos += 8; //handle 64bit x 8 = 512bit _pos += 8; // handle 64bit x 8 = 512bit
} }
for (int32_t i = 0; i < remain; ++i) { for (int32_t i = 0; i < remain; ++i) {
p[_pos++] = prevValue; p[_pos++] = prevValue;
} }
#endif #endif
} else if (tsSIMDEnable && tsAVX2Supported) { } else if (tsSIMDEnable && tsAVX2Supported) {
@ -109,7 +109,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements,
p[_pos++] = prevValue; p[_pos++] = prevValue;
} }
} else { // alternative implementation without SIMD instructions. } else { // alternative implementation without SIMD instructions.
for (int32_t i = 0; i < elems && count < nelements; i++, count++) { for (int32_t i = 0; i < elems && count < nelements; i++, count++) {
p[_pos++] = prevValue; p[_pos++] = prevValue;
v += bit; v += bit;
@ -117,69 +117,67 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements,
} }
} else { } else {
if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) {
#if __AVX512F__ #if __AVX512F__
__m512i sum_mask1 = _mm512_set_epi64(6, 6, 4, 4, 2, 2, 0, 0); __m512i sum_mask1 = _mm512_set_epi64(6, 6, 4, 4, 2, 2, 0, 0);
__m512i sum_mask2 = _mm512_set_epi64(5, 5, 5, 5, 1, 1, 1, 1); __m512i sum_mask2 = _mm512_set_epi64(5, 5, 5, 5, 1, 1, 1, 1);
__m512i sum_mask3 = _mm512_set_epi64(3, 3, 3, 3, 3, 3, 3, 3); __m512i sum_mask3 = _mm512_set_epi64(3, 3, 3, 3, 3, 3, 3, 3);
__m512i base = _mm512_set1_epi64(w); __m512i base = _mm512_set1_epi64(w);
__m512i maskVal = _mm512_set1_epi64(mask); __m512i maskVal = _mm512_set1_epi64(mask);
__m512i shiftBits = _mm512_set_epi64(bit * 7 + 4, bit * 6 + 4, bit * 5 + 4, bit * 4 + 4, bit * 3 + 4, bit * 2 + 4, bit + 4, 4); __m512i shiftBits = _mm512_set_epi64(bit * 7 + 4, bit * 6 + 4, bit * 5 + 4, bit * 4 + 4, bit * 3 + 4,
bit * 2 + 4, bit + 4, 4);
__m512i inc = _mm512_set1_epi64(bit << 3); __m512i inc = _mm512_set1_epi64(bit << 3);
for (int32_t i = 0; i < batch; ++i) { for (int32_t i = 0; i < batch; ++i) {
__m512i after = _mm512_srlv_epi64(base, shiftBits);
__m512i zigzagVal = _mm512_and_si512(after, maskVal);
__m512i after = _mm512_srlv_epi64(base, shiftBits); // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1)))
__m512i zigzagVal = _mm512_and_si512(after, maskVal); __m512i signmask = _mm512_and_si512(_mm512_set1_epi64(1), zigzagVal);
signmask = _mm512_sub_epi64(_mm512_setzero_si512(), signmask);
__m512i delta = _mm512_xor_si512(_mm512_srli_epi64(zigzagVal, 1), signmask);
// ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) // calculate the cumulative sum (prefix sum) for each number
__m512i signmask = _mm512_and_si512(_mm512_set1_epi64(1), zigzagVal); // decode[0] = prevValue + final[0]
signmask = _mm512_sub_epi64(_mm512_setzero_si512(), signmask); // decode[1] = decode[0] + final[1] -----> prevValue + final[0] + final[1]
__m512i delta = _mm512_xor_si512(_mm512_srli_epi64(zigzagVal, 1), signmask); // decode[2] = decode[1] + final[2] -----> prevValue + final[0] + final[1] + final[2]
// decode[3] = decode[2] + final[3] -----> prevValue + final[0] + final[1] + final[2] + final[3]
// calculate the cumulative sum (prefix sum) for each number // 7 6 5 4 3 2 1
// decode[0] = prevValue + final[0] // 0 D7 D6 D5 D4 D3 D2 D1
// decode[1] = decode[0] + final[1] -----> prevValue + final[0] + final[1] // D0 D6 0 D4 0 D2 0 D0
// decode[2] = decode[1] + final[2] -----> prevValue + final[0] + final[1] + final[2] // 0 D7+D6 D6 D5+D4 D4 D3+D2 D2
// decode[3] = decode[2] + final[3] -----> prevValue + final[0] + final[1] + final[2] + final[3] // D1+D0 D0 13 6 9 4 5 2
// 1 0
__m512i prev = _mm512_set1_epi64(prevValue);
__m512i cum_sum = _mm512_add_epi64(delta, _mm512_maskz_permutexvar_epi64(0xaa, sum_mask1, delta));
cum_sum = _mm512_add_epi64(cum_sum, _mm512_maskz_permutexvar_epi64(0xcc, sum_mask2, cum_sum));
cum_sum = _mm512_add_epi64(cum_sum, _mm512_maskz_permutexvar_epi64(0xf0, sum_mask3, cum_sum));
// 13 6 9 4 5 2 1
// 0 D7,D6 D6 D5,D4 D4 D3,D2 D2
// D1,D0 D0 +D5,D4 D5,D4, 0 0 D1,D0 D1,D0
//0 0 D7~D4 D6~D4 D5~D4 D4 D3~D0 D2~D0
// D1~D0 D0 22 15 9 4 6 3
// 1 0
//
// D3~D0 D3~D0 D3~D0 D3~D0 0 0 0
// 0 28 21 15 10 6 3 1
// 0
//7 6 5 4 3 2 1 0 cum_sum = _mm512_add_epi64(cum_sum, prev);
//D7 D6 D5 D4 D3 D2 D1 D0 _mm512_storeu_si512((__m512i *)&p[_pos], cum_sum);
//D6 0 D4 0 D2 0 D0 0
//D7+D6 D6 D5+D4 D4 D3+D2 D2 D1+D0 D0
//13 6 9 4 5 2 1 0
__m512i prev = _mm512_set1_epi64(prevValue);
__m512i cum_sum = _mm512_add_epi64(delta, _mm512_maskz_permutexvar_epi64(0xaa, sum_mask1, delta));
cum_sum = _mm512_add_epi64(cum_sum, _mm512_maskz_permutexvar_epi64(0xcc, sum_mask2, cum_sum));
cum_sum = _mm512_add_epi64(cum_sum, _mm512_maskz_permutexvar_epi64(0xf0, sum_mask3, cum_sum));
//13 6 9 4 5 2 1 0
//D7,D6 D6 D5,D4 D4 D3,D2 D2 D1,D0 D0
//+D5,D4 D5,D4, 0 0 D1,D0 D1,D0 0 0
//D7~D4 D6~D4 D5~D4 D4 D3~D0 D2~D0 D1~D0 D0
//22 15 9 4 6 3 1 0
//
//D3~D0 D3~D0 D3~D0 D3~D0 0 0 0 0
//28 21 15 10 6 3 1 0
cum_sum = _mm512_add_epi64(cum_sum, prev);
_mm512_storeu_si512((__m512i *)&p[_pos], cum_sum);
shiftBits = _mm512_add_epi64(shiftBits, inc);
prevValue = p[_pos + 7];
_pos += 8;
shiftBits = _mm512_add_epi64(shiftBits, inc);
prevValue = p[_pos + 7];
_pos += 8;
} }
// handle the remain value // handle the remain value
for (int32_t i = 0; i < remain; i++) { for (int32_t i = 0; i < remain; i++) {
zigzag_value = ((w >> (v + (batch * bit * 8))) & mask); zigzag_value = ((w >> (v + (batch * bit * 8))) & mask);
prevValue += ZIGZAG_DECODE(int64_t, zigzag_value); prevValue += ZIGZAG_DECODE(int64_t, zigzag_value);
p[_pos++] = prevValue; p[_pos++] = prevValue;
v += bit; v += bit;
} }
#endif #endif
} else if (tsSIMDEnable && tsAVX2Supported) { } else if (tsSIMDEnable && tsAVX2Supported) {
@ -255,7 +253,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements,
} }
} break; } break;
case TSDB_DATA_TYPE_INT: { case TSDB_DATA_TYPE_INT: {
int32_t* p = (int32_t*) output; int32_t *p = (int32_t *)output;
if (selector == 0 || selector == 1) { if (selector == 0 || selector == 1) {
for (int32_t i = 0; i < elems && count < nelements; i++, count++) { for (int32_t i = 0; i < elems && count < nelements; i++, count++) {
@ -272,7 +270,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements,
} }
} break; } break;
case TSDB_DATA_TYPE_SMALLINT: { case TSDB_DATA_TYPE_SMALLINT: {
int16_t* p = (int16_t*) output; int16_t *p = (int16_t *)output;
if (selector == 0 || selector == 1) { if (selector == 0 || selector == 1) {
for (int32_t i = 0; i < elems && count < nelements; i++, count++) { for (int32_t i = 0; i < elems && count < nelements; i++, count++) {
@ -317,7 +315,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements,
int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output) { int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output) {
#if __AVX512F__ #if __AVX512F__
// todo add it // todo add it
#endif #endif
return 0; return 0;
} }
@ -336,12 +334,12 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen
int32_t ipos = 1, opos = 0; int32_t ipos = 1, opos = 0;
#if __AVX2__ #if __AVX2__
__m128i prevVal = _mm_setzero_si128(); __m128i prevVal = _mm_setzero_si128();
__m128i prevDelta = _mm_setzero_si128(); __m128i prevDelta = _mm_setzero_si128();
int32_t batch = nelements >> 1; int32_t batch = nelements >> 1;
int32_t remainder = nelements & 0x01; int32_t remainder = nelements & 0x01;
// __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; // __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff};
int32_t i = 0; int32_t i = 0;
if (batch > 1) { if (batch > 1) {
@ -354,14 +352,14 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen
__m128i data1 = _mm_setzero_si128(); __m128i data1 = _mm_setzero_si128();
if (nbytes1 > 0) { if (nbytes1 > 0) {
int64_t tmp = 0; int64_t tmp = 0;
memcpy(&tmp, (const void*) (input + ipos), nbytes1); memcpy(&tmp, (const void *)(input + ipos), nbytes1);
data1 = _mm_set1_epi64x(tmp); data1 = _mm_set1_epi64x(tmp);
} }
__m128i data2 = _mm_setzero_si128(); __m128i data2 = _mm_setzero_si128();
if (nbytes2 > 0) { if (nbytes2 > 0) {
int64_t tmp = 0; int64_t tmp = 0;
memcpy(&tmp, (const void*) (input + ipos + nbytes1), nbytes2); memcpy(&tmp, (const void *)(input + ipos + nbytes1), nbytes2);
data2 = _mm_set1_epi64x(tmp); data2 = _mm_set1_epi64x(tmp);
} }
@ -382,7 +380,7 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen
_mm_storeu_si128((__m128i *)&ostream[opos], finalVal); _mm_storeu_si128((__m128i *)&ostream[opos], finalVal);
// keep the previous value // keep the previous value
prevVal = _mm_shuffle_epi32 (finalVal, 0xEE); prevVal = _mm_shuffle_epi32(finalVal, 0xEE);
// keep the previous delta of delta, for the first item // keep the previous delta of delta, for the first item
prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE); prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE);
@ -393,7 +391,7 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen
} }
// the remain // the remain
for(; i < batch; ++i) { for (; i < batch; ++i) {
uint8_t flags = input[ipos++]; uint8_t flags = input[ipos++];
int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7
@ -402,14 +400,14 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen
__m128i data1 = _mm_setzero_si128(); __m128i data1 = _mm_setzero_si128();
if (nbytes1 > 0) { if (nbytes1 > 0) {
int64_t dd = 0; int64_t dd = 0;
memcpy(&dd, (const void*) (input + ipos), nbytes1); memcpy(&dd, (const void *)(input + ipos), nbytes1);
data1 = _mm_loadu_si64(&dd); data1 = _mm_loadu_si64(&dd);
} }
__m128i data2 = _mm_setzero_si128(); __m128i data2 = _mm_setzero_si128();
if (nbytes2 > 0) { if (nbytes2 > 0) {
int64_t dd = 0; int64_t dd = 0;
memcpy(&dd, (const void*) (input + ipos + nbytes1), nbytes2); memcpy(&dd, (const void *)(input + ipos + nbytes1), nbytes2);
data2 = _mm_loadu_si64(&dd); data2 = _mm_loadu_si64(&dd);
} }
@ -433,10 +431,10 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen
_mm_storeu_si128((__m128i *)&ostream[opos], finalVal); _mm_storeu_si128((__m128i *)&ostream[opos], finalVal);
// keep the previous value // keep the previous value
prevVal = _mm_shuffle_epi32 (finalVal, 0xEE); prevVal = _mm_shuffle_epi32(finalVal, 0xEE);
// keep the previous delta of delta // keep the previous delta of delta
prevDelta = _mm_shuffle_epi32 (deltaCurrent, 0xEE); prevDelta = _mm_shuffle_epi32(deltaCurrent, 0xEE);
opos += 2; opos += 2;
ipos += nbytes1 + nbytes2; ipos += nbytes1 + nbytes2;
@ -478,8 +476,8 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem
#if __AVX512VL__ #if __AVX512VL__
__m128i prevVal = _mm_setzero_si128(); __m128i prevVal = _mm_setzero_si128();
__m128i prevDelta = _mm_setzero_si128(); __m128i prevDelta = _mm_setzero_si128();
int32_t numOfBatch = nelements >> 1; int32_t numOfBatch = nelements >> 1;
int32_t remainder = nelements & 0x01; int32_t remainder = nelements & 0x01;
@ -493,8 +491,8 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem
int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7
int8_t nbytes2 = (flags >> 4) & INT8MASK(4); int8_t nbytes2 = (flags >> 4) & INT8MASK(4);
__m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void *)(input + ipos));
__m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void *)(input + ipos + nbytes1));
data2 = _mm_broadcastq_epi64(data2); data2 = _mm_broadcastq_epi64(data2);
__m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03);
@ -513,7 +511,7 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem
_mm_storeu_si128((__m128i *)&ostream[opos], val); _mm_storeu_si128((__m128i *)&ostream[opos], val);
// keep the previous value // keep the previous value
prevVal = _mm_shuffle_epi32 (val, 0xEE); prevVal = _mm_shuffle_epi32(val, 0xEE);
// keep the previous delta of delta, for the first item // keep the previous delta of delta, for the first item
prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE); prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE);
@ -524,14 +522,14 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem
} }
// the remain // the remain
for(; i < numOfBatch; ++i) { for (; i < numOfBatch; ++i) {
uint8_t flags = input[ipos++]; uint8_t flags = input[ipos++];
int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7
int8_t nbytes2 = (flags >> 4) & INT8MASK(4); int8_t nbytes2 = (flags >> 4) & INT8MASK(4);
__m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void *)(input + ipos));
__m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void *)(input + ipos + nbytes1));
data2 = _mm_broadcastq_epi64(data2); data2 = _mm_broadcastq_epi64(data2);
__m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03);
@ -550,7 +548,7 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem
_mm_storeu_si128((__m128i *)&ostream[opos], val); _mm_storeu_si128((__m128i *)&ostream[opos], val);
// keep the previous value // keep the previous value
prevVal = _mm_shuffle_epi32 (val, 0xEE); prevVal = _mm_shuffle_epi32(val, 0xEE);
// keep the previous delta of delta // keep the previous delta of delta
__m128i delta = _mm_add_epi64(_mm_slli_si128(deltaOfDelta, 8), deltaOfDelta); __m128i delta = _mm_add_epi64(_mm_slli_si128(deltaOfDelta, 8), deltaOfDelta);

2
source/util/src/terror.c
View File

@ -674,7 +674,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COL_PK_TYPE, "primary key column
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PK_OP, "primary key column can not be added, modified, and dropped") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PK_OP, "primary key column can not be added, modified, and dropped")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL, "Primary key column should not be null") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL, "Primary key column should not be null")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE, "Primary key column should not be none") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE, "Primary key column should not be none")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_ERROR, "Tbanme duplicated or not set") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_ERROR, "Pseudo tag tbname not set")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_DUPLICATED, "Table name duplicated") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_DUPLICATED, "Table name duplicated")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TAG_NAME_DUPLICATED, "Tag name duplicated") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TAG_NAME_DUPLICATED, "Tag name duplicated")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error")