xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/udf' of github.com:taosdata/TDengine into feature/udf

This commit is contained in:
slzhou 2022-05-19 20:56:40 +08:00
parent c0fc53a9a8 5a3f6b11e7
commit b5a419c25a
80 changed files with 2536 additions and 1988 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
Jenkinsfile2
View File

@ -117,27 +117,29 @@ def pre_test(){
def pre_test_win(){ def pre_test_win(){
bat ''' bat '''
hostname hostname
ipconfig
set
date /t date /t
time /t time /t
taskkill /f /t /im python.exe
taskkill /f /t /im bash.exe
rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug
exit 0
''' '''
bat ''' bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git reset --hard git reset --hard
git fetch || git fetch git fetch || git fetch
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git reset --hard git reset --hard
git fetch || git fetch git fetch || git fetch
git checkout -f
''' '''
script { script {
if (env.CHANGE_TARGET == 'master') { if (env.CHANGE_TARGET == 'master') {
bat ''' bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git checkout master git checkout master
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git checkout master git checkout master
''' '''
@ -145,6 +147,8 @@ def pre_test_win(){
bat ''' bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git checkout 2.0 git checkout 2.0
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git checkout 2.0 git checkout 2.0
''' '''
@ -152,6 +156,8 @@ def pre_test_win(){
bat ''' bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git checkout 3.0 git checkout 3.0
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git checkout 3.0 git checkout 3.0
''' '''
@ -159,6 +165,8 @@ def pre_test_win(){
bat ''' bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git checkout develop git checkout develop
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git checkout develop git checkout develop
''' '''
@ -169,30 +177,52 @@ def pre_test_win(){
bat ''' bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git pull git pull
git log -5 '''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git pull git pull
git fetch origin +refs/pull/${CHANGE_ID}/merge '''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git fetch origin +refs/pull/%CHANGE_ID%/merge
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git checkout -qf FETCH_HEAD git checkout -qf FETCH_HEAD
git log -5
''' '''
} else if (env.CHANGE_URL =~ /\/TDinternal\//) { } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
bat ''' bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git pull git pull
git fetch origin +refs/pull/${CHANGE_ID}/merge '''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git fetch origin +refs/pull/%CHANGE_ID%/merge
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git checkout -qf FETCH_HEAD git checkout -qf FETCH_HEAD
git log -5 '''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git pull git pull
git log -5
''' '''
} else { } else {
sh ''' bat '''
echo "unmatched reposiotry ${CHANGE_URL}" echo "unmatched reposiotry %CHANGE_URL%"
''' '''
} }
} }
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
git branch
git log -5
'''
bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git branch
git log -5
'''
bat ''' bat '''
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
git submodule update --init --recursive git submodule update --init --recursive
@ -205,10 +235,15 @@ def pre_test_build_win() {
cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
mkdir debug mkdir debug
cd debug cd debug
time /t
call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64
set CL=/MP8 set CL=/MP8
cmake .. -G "NMake Makefiles JOM" echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake"
jom -j 4 || exit 8 time /t
cmake .. -G "NMake Makefiles JOM" || exit 7
echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6"
time /t
jom -j 6 || exit 8
time /t time /t
''' '''
return 1 return 1
@ -226,6 +261,13 @@ pipeline {
stages { stages {
stage('run test') { stage('run test') {
parallel { parallel {
stage('windows test') {
agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "}
steps {
pre_test_win()
pre_test_build_win()
}
}
stage('linux test') { stage('linux test') {
agent{label " slave3_0 || slave15 || slave16 || slave17 "} agent{label " slave3_0 || slave15 || slave16 || slave17 "}
options { skipDefaultCheckout() } options { skipDefaultCheckout() }

6
include/common/tmsgcb.h
View File

@ -60,9 +60,9 @@ typedef struct {
ReportStartup reportStartupFp; ReportStartup reportStartupFp;
} SMsgCb; } SMsgCb;
void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb); void tmsgSetDefault(const SMsgCb* msgcb);
int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg); int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg);
int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype);
int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg);
void tmsgSendRsp(SRpcMsg* pMsg); void tmsgSendRsp(SRpcMsg* pMsg);
void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet); void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet);

2
include/common/ttypes.h
View File

@ -179,6 +179,8 @@ typedef struct {
} \ } \
} while (0) } while (0)
//TODO: use varchar(0) to represent NULL type
#define IS_NULL_TYPE(_t) ((_t) == TSDB_DATA_TYPE_NULL)
#define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) #define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT)
#define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT)
#define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE)

4
include/libs/executor/executor.h
View File

@ -81,7 +81,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO
* @param isAdd * @param isAdd
* @return * @return
*/ */
int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd); int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd);
/** /**
* Create the exec task object according to task json * Create the exec task object according to task json
@ -169,7 +169,7 @@ int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type);
void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes); int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
include/libs/nodes/nodes.h
View File

@ -208,6 +208,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_SORT,
QUERY_NODE_PHYSICAL_PLAN_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_FILL,
QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW,
QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW,

1
include/libs/nodes/querynodes.h
View File

@ -248,6 +248,7 @@ typedef struct SSetOperator {
SNode* pRight; SNode* pRight;
SNodeList* pOrderByList; // SOrderByExprNode SNodeList* pOrderByList; // SOrderByExprNode
SNode* pLimit; SNode* pLimit;
char stmtName[TSDB_TABLE_NAME_LEN];
} SSetOperator; } SSetOperator;
typedef enum ESqlClause { typedef enum ESqlClause {

6
include/libs/qcom/query.h
View File

@ -223,19 +223,19 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define qDebug(...) \ #define qDebug(...) \
do { \ do { \
if (qDebugFlag & DEBUG_DEBUG) { \ if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qTrace(...) \ #define qTrace(...) \
do { \ do { \
if (qDebugFlag & DEBUG_TRACE) { \ if (qDebugFlag & DEBUG_TRACE) { \
taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define qDebugL(...) \ #define qDebugL(...) \
do { \ do { \
if (qDebugFlag & DEBUG_DEBUG) { \ if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)

1
include/util/taoserror.h
View File

@ -646,6 +646,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INVALID_ALTER_TABLE TAOS_DEF_ERROR_CODE(0, 0x2649) #define TSDB_CODE_PAR_INVALID_ALTER_TABLE TAOS_DEF_ERROR_CODE(0, 0x2649)
#define TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x264A) #define TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x264A)
#define TSDB_CODE_PAR_INVALID_MODIFY_COL TAOS_DEF_ERROR_CODE(0, 0x264B) #define TSDB_CODE_PAR_INVALID_MODIFY_COL TAOS_DEF_ERROR_CODE(0, 0x264B)
#define TSDB_CODE_PAR_INVALID_TBNAME TAOS_DEF_ERROR_CODE(0, 0x264C)
//planner //planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)

15
source/client/src/tmq.c
View File

@ -830,10 +830,12 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
} }
// init hb timer // init hb timer
if (tmq->hbTimer == NULL) {
tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer);
}
// init auto commit timer // init auto commit timer
if (tmq->autoCommit) { if (tmq->autoCommit && tmq->commitTimer == NULL) {
tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer); tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer);
} }
@ -1456,9 +1458,18 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
if (tmq->status == TMQ_CONSUMER_STATUS__READY) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL);
if (rsp == TMQ_RESP_ERR__SUCCESS) {
// TODO: free resources
return TMQ_RESP_ERR__SUCCESS;
} else {
return TMQ_RESP_ERR__FAIL;
}
tmq_list_t* lst = tmq_list_new(); tmq_list_t* lst = tmq_list_new();
tmq_resp_err_t rsp = tmq_subscribe(tmq, lst); rsp = tmq_subscribe(tmq, lst);
tmq_list_destroy(lst); tmq_list_destroy(lst);
if (rsp == TMQ_RESP_ERR__SUCCESS) { if (rsp == TMQ_RESP_ERR__SUCCESS) {
// TODO: free resources // TODO: free resources
return TMQ_RESP_ERR__SUCCESS; return TMQ_RESP_ERR__SUCCESS;

28
source/common/src/tmsgcb.c
View File

@ -17,46 +17,46 @@
#include "tmsgcb.h" #include "tmsgcb.h"
#include "taoserror.h" #include "taoserror.h"
static SMsgCb tsDefaultMsgCb; static SMsgCb defaultMsgCb;
void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; }
int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg) { int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) {
PutToQueueFp fp = pMsgCb->queueFps[qtype]; PutToQueueFp fp = msgcb->queueFps[qtype];
return (*fp)(pMsgCb->mgmt, pMsg); return (*fp)(msgcb->mgmt, pMsg);
} }
int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) {
GetQueueSizeFp fp = pMsgCb->qsizeFp; GetQueueSizeFp fp = msgcb->qsizeFp;
return (*fp)(pMsgCb->mgmt, vgId, qtype); return (*fp)(msgcb->mgmt, vgId, qtype);
} }
int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) {
SendReqFp fp = tsDefaultMsgCb.sendReqFp; SendReqFp fp = defaultMsgCb.sendReqFp;
return (*fp)(epSet, pMsg); return (*fp)(epSet, pMsg);
} }
void tmsgSendRsp(SRpcMsg* pMsg) { void tmsgSendRsp(SRpcMsg* pMsg) {
SendRspFp fp = tsDefaultMsgCb.sendRspFp; SendRspFp fp = defaultMsgCb.sendRspFp;
return (*fp)(pMsg); return (*fp)(pMsg);
} }
void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) {
SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; SendRedirectRspFp fp = defaultMsgCb.sendRedirectRspFp;
(*fp)(pMsg, pNewEpSet); (*fp)(pMsg, pNewEpSet);
} }
void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) {
RegisterBrokenLinkArgFp fp = tsDefaultMsgCb.registerBrokenLinkArgFp; RegisterBrokenLinkArgFp fp = defaultMsgCb.registerBrokenLinkArgFp;
(*fp)(pMsg); (*fp)(pMsg);
} }
void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) {
ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp; ReleaseHandleFp fp = defaultMsgCb.releaseHandleFp;
(*fp)(pHandle, type); (*fp)(pHandle, type);
} }
void tmsgReportStartup(const char* name, const char* desc) { void tmsgReportStartup(const char* name, const char* desc) {
ReportStartup fp = tsDefaultMsgCb.reportStartupFp; ReportStartup fp = defaultMsgCb.reportStartupFp;
(*fp)(name, desc); (*fp)(name, desc);
} }

2
source/common/src/trow.c
View File

@ -1063,7 +1063,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl
int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) {
if (isAllRowsNone(pCol)) { if (isAllRowsNone(pCol)) {
pVal->valType = TD_VTYPE_NULL; pVal->valType = TD_VTYPE_NONE;
#ifdef TD_SUPPORT_READ2 #ifdef TD_SUPPORT_READ2
pVal->val = (void *)getNullValue(pCol->type); pVal->val = (void *)getNullValue(pCol->type);
#else #else

4
source/dnode/mgmt/mgmt_bnode/src/bmWorker.c
View File

@ -38,9 +38,9 @@ static void bmSendErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) {
static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) { static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) {
SRpcMsg rsp = { SRpcMsg rsp = {
.code = code, .code = code,
.info = pMsg->info,
.pCont = pMsg->info.rsp, .pCont = pMsg->info.rsp,
.contLen = pMsg->info.rspLen, .contLen = pMsg->info.rspLen,
.info = pMsg->info,
}; };
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
@ -63,7 +63,7 @@ static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
bmSendRsp(pMsg, code); bmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }

7
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -86,7 +86,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527}; SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527};
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
dTrace("send status msg to mnode, app:%p", rpcMsg.info.ahandle); dTrace("send status msg to mnode");
SEpSet epSet = {0}; SEpSet epSet = {0};
dmGetMnodeEpSet(pMgmt->pData, &epSet); dmGetMnodeEpSet(pMgmt->pData, &epSet);
@ -116,8 +116,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
SServerStatusRsp statusRsp = {0}; SServerStatusRsp statusRsp = {0};
SMonMloadInfo minfo = {0}; SMonMloadInfo minfo = {0};
dmGetMnodeLoads(pMgmt, &minfo); dmGetMnodeLoads(pMgmt, &minfo);
if (minfo.isMnode && minfo.load.syncState != TAOS_SYNC_STATE_LEADER && if (minfo.isMnode && minfo.load.syncState == TAOS_SYNC_STATE_ERROR) {
minfo.load.syncState != TAOS_SYNC_STATE_CANDIDATE) {
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED;
snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState));
return; return;
@ -127,7 +126,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
dmGetVnodeLoads(pMgmt, &vinfo); dmGetVnodeLoads(pMgmt, &vinfo);
for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) {
SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i);
if (pLoad->syncState != TAOS_SYNC_STATE_LEADER && pLoad->syncState != TAOS_SYNC_STATE_FOLLOWER) { if (pLoad->syncState == TAOS_SYNC_STATE_ERROR) {
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED;
snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pLoad->vgId, snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pLoad->vgId,
syncStr(pLoad->syncState)); syncStr(pLoad->syncState));

4
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -153,14 +153,14 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
if (code != 0 && terrno != 0) code = terrno; if (code != 0 && terrno != 0) code = terrno;
SRpcMsg rsp = { SRpcMsg rsp = {
.code = code, .code = code,
.info = pMsg->info,
.pCont = pMsg->info.rsp, .pCont = pMsg->info.rsp,
.contLen = pMsg->info.rspLen, .contLen = pMsg->info.rspLen,
.info = pMsg->info,
}; };
rpcSendResponse(&rsp); rpcSendResponse(&rsp);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }

39
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -19,9 +19,9 @@
static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) {
SRpcMsg rsp = { SRpcMsg rsp = {
.code = code, .code = code,
.info = pMsg->info,
.pCont = pMsg->info.rsp, .pCont = pMsg->info.rsp,
.contLen = pMsg->info.rspLen, .contLen = pMsg->info.rspLen,
.info = pMsg->info,
}; };
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
@ -29,7 +29,7 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) {
static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SMnodeMgmt *pMgmt = pInfo->ahandle; SMnodeMgmt *pMgmt = pInfo->ahandle;
int32_t code = -1; int32_t code = -1;
dTrace("msg:%p, get from mnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); dTrace("msg:%p, get from mnode queue", pMsg);
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_DND_ALTER_MNODE: case TDMT_DND_ALTER_MNODE:
@ -51,7 +51,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
mmSendRsp(pMsg, code); mmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -73,7 +73,7 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
} }
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -84,13 +84,20 @@ static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) {
return 0; return 0;
} }
int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); } int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg);
}
int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); } int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg);
}
int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); } int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg);
}
int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg);
} }
int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
@ -101,25 +108,25 @@ static inline int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc)
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
if (pMsg == NULL) return -1; if (pMsg == NULL) return -1;
dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pRpc); return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pMsg);
} }
int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pRpc); return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pMsg);
} }
int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutRpcMsgToWorker(&pMgmt->readWorker, pRpc); return mmPutRpcMsgToWorker(&pMgmt->readWorker, pMsg);
} }
int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pRpc); } int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg); }
int32_t mmStartWorker(SMnodeMgmt *pMgmt) { int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
SSingleWorkerCfg qCfg = { SSingleWorkerCfg qCfg = {

8
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -44,7 +44,7 @@ static void qmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
qmSendRsp(pMsg, code); qmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -60,7 +60,7 @@ static void qmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
qmSendRsp(pMsg, code); qmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -76,7 +76,7 @@ static void qmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
qmSendRsp(pMsg, code); qmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -105,7 +105,7 @@ static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SR
return -1; return -1;
} }
dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;

12
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -19,9 +19,9 @@
static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) { static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) {
SRpcMsg rsp = { SRpcMsg rsp = {
.code = code, .code = code,
.info = pMsg->info,
.pCont = pMsg->info.rsp, .pCont = pMsg->info.rsp,
.contLen = pMsg->info.rspLen, .contLen = pMsg->info.rspLen,
.info = pMsg->info,
}; };
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
@ -44,7 +44,7 @@ static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
smSendRsp(pMsg, code); smSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -166,7 +166,7 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1; return -1;
} }
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -174,7 +174,7 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t smPutNodeMsgToMonitorQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToMonitorQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->monitorWorker; SSingleWorker *pWorker = &pMgmt->monitorWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -187,7 +187,7 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1; return -1;
} }
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -195,7 +195,7 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->sharedWorker; SSingleWorker *pWorker = &pMgmt->sharedWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }

70
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -51,7 +51,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
break; break;
default: default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED; terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dError("msg:%p, not processed in vnode-mgmt/monitor queue", pMsg); dError("msg:%p, not processed in vnode queue", pMsg);
} }
if (msgType & 1u) { if (msgType & 1u) {
@ -59,7 +59,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -67,13 +67,13 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SVnodeObj *pVnode = pInfo->ahandle; SVnodeObj *pVnode = pInfo->ahandle;
dTrace("msg:%p, will be processed in vnode-query queue", pMsg); dTrace("msg:%p, get from vnode-query queue", pMsg);
int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg);
if (code != 0) { if (code != 0) {
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -82,13 +82,13 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SVnodeObj *pVnode = pInfo->ahandle; SVnodeObj *pVnode = pInfo->ahandle;
dTrace("msg:%p, will be processed in vnode-fetch queue", pMsg); dTrace("msg:%p, get from vnode-fetch queue", pMsg);
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
if (code != 0) { if (code != 0) {
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -96,7 +96,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle; SVnodeObj *pVnode = pInfo->ahandle;
SRpcMsg rsp;
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
if (pArray == NULL) { if (pArray == NULL) {
@ -108,7 +107,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
SRpcMsg *pMsg = NULL; SRpcMsg *pMsg = NULL;
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
dTrace("msg:%p, will be processed in vnode-write queue", pMsg); dTrace("msg:%p, get from vnode-write queue", pMsg);
if (taosArrayPush(pArray, &pMsg) == NULL) { if (taosArrayPush(pArray, &pMsg) == NULL) {
dTrace("msg:%p, failed to process since %s", pMsg, terrstr()); dTrace("msg:%p, failed to process since %s", pMsg, terrstr());
vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
@ -116,21 +115,12 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
} }
for (int i = 0; i < taosArrayGetSize(pArray); i++) { for (int i = 0; i < taosArrayGetSize(pArray); i++) {
SRpcMsg *pMsg; SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
SRpcMsg *pRpc; SRpcMsg rsp = {.info = pMsg->info, .pCont = NULL, .contLen = 0};
pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false);
pRpc = pMsg;
rsp.info = pRpc->info;
rsp.pCont = NULL;
rsp.contLen = 0;
int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false);
if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
// rsp.code = TSDB_CODE_SYN_NOT_LEADER; dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId);
// tmsgSendRsp(&rsp);
dTrace("syncPropose not leader redirect, vgId:%d ", syncGetVgId(vnodeGetSyncHandle(pVnode->pImpl)));
rsp.code = TSDB_CODE_RPC_REDIRECT; rsp.code = TSDB_CODE_RPC_REDIRECT;
SEpSet newEpSet; SEpSet newEpSet;
syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet);
@ -181,7 +171,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
// apply data into tsdb // apply data into tsdb
if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) { if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
rsp.code = terrno; rsp.code = terrno;
dTrace("vnodeProcessWriteReq error, code:%d", terrno); dTrace("msg:%p, process write error since %s", pMsg, terrstr());
} }
syncApplyMsgDestroy(pSyncApplyMsg); syncApplyMsgDestroy(pSyncApplyMsg);
@ -215,7 +205,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
SRpcMsg rsp = {0}; SRpcMsg rsp = {0};
rsp.code = terrno; rsp.code = terrno;
rsp.info = pMsg->info; rsp.info = pMsg->info;
dTrace("vmProcessSyncQueue error, code:%d", terrno); dTrace("msg:%p, process sync queue error since code:%s", pMsg, terrstr());
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
} }
@ -232,13 +222,13 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
for (int32_t i = 0; i < numOfMsgs; ++i) { for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pMsg); taosGetQitem(qall, (void **)&pMsg);
dTrace("msg:%p, will be processed in vnode-merge queue", pMsg); dTrace("msg:%p, get from vnode-merge queue", pMsg);
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
if (code != 0) { if (code != 0) {
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -255,29 +245,29 @@ static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) { if (pVnode == NULL) {
dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr()); dError("vgId:%d, failed to put msg:%p into vnode-queue since %s", pHead->vgId, pMsg, terrstr());
return terrno != 0 ? terrno : -1; return terrno != 0 ? terrno : -1;
} }
switch (qtype) { switch (qtype) {
case QUERY_QUEUE: case QUERY_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-query queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pQueryQ, pMsg); taosWriteQitem(pVnode->pQueryQ, pMsg);
break; break;
case FETCH_QUEUE: case FETCH_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-fetch queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pFetchQ, pMsg); taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case WRITE_QUEUE: case WRITE_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-write queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pWriteQ, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg);
break; break;
case SYNC_QUEUE: case SYNC_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-sync queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pSyncQ, pMsg); taosWriteQitem(pVnode->pSyncQ, pMsg);
break; break;
case MERGE_QUEUE: case MERGE_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-merge queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pMergeQ, pMsg); taosWriteQitem(pVnode->pMergeQ, pMsg);
break; break;
default: default:
@ -312,7 +302,7 @@ int32_t vmPutNodeMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->mgmtWorker; SSingleWorker *pWorker = &pMgmt->mgmtWorker;
dTrace("msg:%p, will be put into vnode-mgmt queue, worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into vnode-mgmt worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -320,7 +310,7 @@ int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->monitorWorker; SSingleWorker *pWorker = &pMgmt->monitorWorker;
dTrace("msg:%p, will be put into vnode-monitor queue, worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into vnode-monitor worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -335,32 +325,30 @@ static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType q
int32_t code = 0; int32_t code = 0;
if (pMsg != NULL) { if (pMsg != NULL) {
dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
// if (pMsg->handle != NULL) assert(pMsg->refId != 0);
switch (qtype) { switch (qtype) {
case WRITE_QUEUE: case WRITE_QUEUE:
dTrace("msg:%p, will be put into vnode-write queue", pMsg); dTrace("msg:%p, create and put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pWriteQ, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg);
break; break;
case QUERY_QUEUE: case QUERY_QUEUE:
dTrace("msg:%p, will be put into vnode-query queue", pMsg); dTrace("msg:%p, create and put into vnode-query queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pQueryQ, pMsg); taosWriteQitem(pVnode->pQueryQ, pMsg);
break; break;
case FETCH_QUEUE: case FETCH_QUEUE:
dTrace("msg:%p, will be put into vnode-fetch queue", pMsg); dTrace("msg:%p, create and put into vnode-fetch queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pFetchQ, pMsg); taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case APPLY_QUEUE: case APPLY_QUEUE:
dTrace("msg:%p, will be put into vnode-apply queue", pMsg); dTrace("msg:%p, create and put into vnode-apply queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pApplyQ, pMsg); taosWriteQitem(pVnode->pApplyQ, pMsg);
break; break;
case MERGE_QUEUE: case MERGE_QUEUE:
dTrace("msg:%p, will be put into vnode-merge queue", pMsg); dTrace("msg:%p, create and put into vnode-merge queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pMergeQ, pMsg); taosWriteQitem(pVnode->pMergeQ, pMsg);
break; break;
case SYNC_QUEUE: case SYNC_QUEUE:
dTrace("msg:%p, will be put into vnode-sync queue", pMsg); dTrace("msg:%p, create and put into vnode-sync queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pSyncQ, pMsg); taosWriteQitem(pVnode->pSyncQ, pMsg);
break; break;
default: default:

17
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -274,25 +274,24 @@ static void dmGetServerStartupStatus(SDnode *pDnode, SServerStatusRsp *pStatus)
} }
} }
void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pReq) { void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg) {
dDebug("net test req is received"); dDebug("msg:%p, net test req will be processed", pMsg);
SRpcMsg rsp = {.code = 0, .info = pReq->info}; SRpcMsg rsp = {.code = 0, .info = pMsg->info};
rsp.pCont = rpcMallocCont(pReq->contLen); rsp.pCont = rpcMallocCont(pMsg->contLen);
if (rsp.pCont == NULL) { if (rsp.pCont == NULL) {
rsp.code = TSDB_CODE_OUT_OF_MEMORY; rsp.code = TSDB_CODE_OUT_OF_MEMORY;
} else { } else {
rsp.contLen = pReq->contLen; rsp.contLen = pMsg->contLen;
} }
rpcSendResponse(&rsp); rpcSendResponse(&rsp);
} }
void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pReq) { void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg) {
dDebug("server startup status req is received"); dDebug("msg:%p, server startup status req will be processed", pMsg);
SServerStatusRsp statusRsp = {0}; SServerStatusRsp statusRsp = {0};
dmGetServerStartupStatus(pDnode, &statusRsp); dmGetServerStartupStatus(pDnode, &statusRsp);
SRpcMsg rspMsg = {.info = pReq->info}; SRpcMsg rspMsg = {.info = pMsg->info};
int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp);
if (rspLen < 0) { if (rspLen < 0) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;

13
source/dnode/mgmt/node_mgmt/src/dmNodes.c
View File

@ -37,6 +37,7 @@ static int32_t dmCreateShm(SMgmtWrapper *pWrapper) {
dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
return -1; return -1;
} }
dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->proc.shm.id, shmsize); dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->proc.shm.id, shmsize);
return 0; return 0;
} }
@ -60,7 +61,7 @@ static int32_t dmNewProc(SMgmtWrapper *pWrapper, EDndNodeType ntype) {
taosIgnSignal(SIGCHLD); taosIgnSignal(SIGCHLD);
pWrapper->proc.pid = pid; pWrapper->proc.pid = pid;
dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid); dInfo("node:%s, continue running in new pid:%d", pWrapper->name, pid);
return 0; return 0;
} }
@ -77,7 +78,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
if (pWrapper->ntype == DNODE || InChildProc(pWrapper)) { if (pWrapper->ntype == DNODE || InChildProc(pWrapper)) {
tmsgSetDefaultMsgCb(&input.msgCb); tmsgSetDefault(&input.msgCb);
} }
if (OnlyInSingleProc(pWrapper)) { if (OnlyInSingleProc(pWrapper)) {
@ -177,11 +178,11 @@ void dmCloseNode(SMgmtWrapper *pWrapper) {
if (OnlyInParentProc(pWrapper)) { if (OnlyInParentProc(pWrapper)) {
int32_t pid = pWrapper->proc.pid; int32_t pid = pWrapper->proc.pid;
if (pid > 0 && taosProcExist(pid)) { if (pid > 0 && taosProcExist(pid)) {
dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pid); dInfo("node:%s, send kill signal to the child pid:%d", pWrapper->name, pid);
taosKillProc(pid); taosKillProc(pid);
dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pid); dInfo("node:%s, wait for child pid:%d to stop", pWrapper->name, pid);
taosWaitProc(pid); taosWaitProc(pid);
dInfo("node:%s, child process:%d is stopped", pWrapper->name, pid); dInfo("node:%s, child pid:%d is stopped", pWrapper->name, pid);
} }
} }
@ -255,7 +256,7 @@ static void dmWatchNodes(SDnode *pDnode) {
if (!OnlyInParentProc(pWrapper)) continue; if (!OnlyInParentProc(pWrapper)) continue;
if (proc->pid <= 0 || !taosProcExist(proc->pid)) { if (proc->pid <= 0 || !taosProcExist(proc->pid)) {
dError("node:%s, process:%d is killed and needs to restart", pWrapper->name, proc->pid); dError("node:%s, pid:%d is killed and needs to restart", pWrapper->name, proc->pid);
dmCloseProcRpcHandles(&pWrapper->proc); dmCloseProcRpcHandles(&pWrapper->proc);
dmNewProc(pWrapper, ntype); dmNewProc(pWrapper, ntype);
} }

6
source/dnode/mgmt/node_mgmt/src/dmProc.c
View File

@ -162,7 +162,7 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg
return 0; return 0;
} }
static int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) { static inline int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) {
tsem_wait(&queue->sem); tsem_wait(&queue->sem);
taosThreadMutexLock(&queue->mutex); taosThreadMutexLock(&queue->mutex);
@ -412,7 +412,7 @@ void dmCleanupProc(struct SMgmtWrapper *pWrapper) {
SProc *proc = &pWrapper->proc; SProc *proc = &pWrapper->proc;
if (proc->name == NULL) return; if (proc->name == NULL) return;
dDebug("node:%s, start to clean up proc", pWrapper->name); dDebug("node:%s, start to cleanup proc", pWrapper->name);
dmStopProc(proc); dmStopProc(proc);
dmCleanupProcQueue(proc->cqueue); dmCleanupProcQueue(proc->cqueue);
dmCleanupProcQueue(proc->pqueue); dmCleanupProcQueue(proc->pqueue);
@ -433,7 +433,7 @@ void dmCloseProcRpcHandles(SProc *proc) {
SRpcHandleInfo *pInfo = taosHashIterate(proc->hash, NULL); SRpcHandleInfo *pInfo = taosHashIterate(proc->hash, NULL);
while (pInfo != NULL) { while (pInfo != NULL) {
dError("node:%s, the child process dies and send an offline rsp to handle:%p", proc->name, pInfo->handle); dError("node:%s, the child process dies and send an offline rsp to handle:%p", proc->name, pInfo->handle);
SRpcMsg rpcMsg = {.info = *pInfo, .code = TSDB_CODE_NODE_OFFLINE}; SRpcMsg rpcMsg = {.code = TSDB_CODE_NODE_OFFLINE, .info = *pInfo};
rpcSendResponse(&rpcMsg); rpcSendResponse(&rpcMsg);
pInfo = taosHashIterate(proc->hash, pInfo); pInfo = taosHashIterate(proc->hash, pInfo);
} }

67
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -43,8 +43,8 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
return -1; return -1;
} }
dTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name);
pMsg->info.wrapper = pWrapper; pMsg->info.wrapper = pWrapper;
dTrace("msg:%p, will be processed by %s, handle:%p", pMsg, pWrapper->name, pMsg->info.handle);
return (*msgFp)(pWrapper->pMgmt, pMsg); return (*msgFp)(pWrapper->pMgmt, pMsg);
} }
@ -56,8 +56,8 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
SMgmtWrapper *pWrapper = NULL; SMgmtWrapper *pWrapper = NULL;
dTrace("msg:%s is received, handle:%p cont:%p len:%d code:0x%04x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), dTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType),
pRpc->info.handle, pRpc->pCont, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId);
pRpc->info.noResp = 0; pRpc->info.noResp = 0;
pRpc->info.persistHandle = 0; pRpc->info.persistHandle = 0;
pRpc->info.wrapper = NULL; pRpc->info.wrapper = NULL;
@ -344,66 +344,6 @@ void dmCleanupClient(SDnode *pDnode) {
} }
} }
static inline int32_t dmGetHideUserAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
int32_t code = 0;
char pass[TSDB_PASSWORD_LEN + 1] = {0};
if (strcmp(user, INTERNAL_USER) == 0) {
taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
} else if (strcmp(user, TSDB_NETTEST_USER) == 0) {
taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
} else {
code = -1;
}
if (code == 0) {
memcpy(secret, pass, TSDB_PASSWORD_LEN);
*spi = 1;
*encrypt = 0;
*ckey = 0;
}
return code;
}
static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret,
char *ckey) {
if (dmGetHideUserAuth(user, spi, encrypt, secret, ckey) == 0) {
dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
return 0;
}
SAuthReq authReq = {0};
tstrncpy(authReq.user, user, TSDB_USER_LEN);
int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
void *pReq = rpcMallocCont(contLen);
tSerializeSAuthReq(pReq, contLen, &authReq);
SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .info.ahandle = (void *)9528};
SRpcMsg rpcRsp = {0};
SEpSet epSet = {0};
dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt);
dmGetMnodeEpSet(&pDnode->data, &epSet);
dmSendRecv(&epSet, &rpcMsg, &rpcRsp);
if (rpcRsp.code != 0) {
terrno = rpcRsp.code;
dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr());
} else {
SAuthRsp authRsp = {0};
tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp);
memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN);
memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN);
*spi = authRsp.spi;
*encrypt = authRsp.encrypt;
dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi,
authRsp.encrypt);
}
rpcFreeCont(rpcRsp.pCont);
return rpcRsp.code;
}
int32_t dmInitServer(SDnode *pDnode) { int32_t dmInitServer(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans; SDnodeTrans *pTrans = &pDnode->trans;
@ -416,7 +356,6 @@ int32_t dmInitServer(SDnode *pDnode) {
rpcInit.sessions = tsMaxShellConns; rpcInit.sessions = tsMaxShellConns;
rpcInit.connType = TAOS_CONN_SERVER; rpcInit.connType = TAOS_CONN_SERVER;
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.afp = (RpcAfp)dmRetrieveUserAuthInfo;
rpcInit.parent = pDnode; rpcInit.parent = pDnode;
pTrans->serverRpc = rpcOpen(&rpcInit); pTrans->serverRpc = rpcOpen(&rpcInit);

2
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -37,7 +37,7 @@ int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray
void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
#ifdef __cplusplus #ifdef __cplusplus
} }

5
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -627,6 +627,10 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) {
ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);
if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) {
pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
} else {
SArray *tmp = pOldConsumer->rebNewTopics; SArray *tmp = pOldConsumer->rebNewTopics;
pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics;
pNewConsumer->rebNewTopics = tmp; pNewConsumer->rebNewTopics = tmp;
@ -642,6 +646,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->subscribeTime = pNewConsumer->upTime; pOldConsumer->subscribeTime = pNewConsumer->upTime;
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
}
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);

43
source/dnode/mnode/impl/src/mndDb.c
View File

@ -261,8 +261,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) {
sdbRelease(pSdb, pDb); sdbRelease(pSdb, pDb);
} }
static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid) {
bool isRedo) {
STransAction action = {0}; STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
@ -279,49 +278,30 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p
action.msgType = TDMT_DND_CREATE_VNODE; action.msgType = TDMT_DND_CREATE_VNODE;
action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
if (isRedo) {
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
} }
} else {
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
}
return 0; return 0;
} }
static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
bool isRedo) {
STransAction action = {0}; STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) return -1;
action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); void *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen);
if (pReq == NULL) return -1; if (pReq == NULL) return -1;
action.pCont = pReq; action.pCont = pReq;
action.contLen = contLen; action.contLen = contLen;
action.msgType = TDMT_VND_ALTER_VNODE; action.msgType = TDMT_VND_ALTER_VNODE;
if (isRedo) {
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
} }
} else {
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq);
return -1;
}
}
return 0; return 0;
} }
@ -487,7 +467,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
SVnodeGid *pVgid = pVgroup->vnodeGid + vn; SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid) != 0) {
return -1; return -1;
} }
} }
@ -726,12 +706,9 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) {
for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup) != 0) {
SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) {
return -1; return -1;
} }
}
} else { } else {
SVgObj newVgroup = {0}; SVgObj newVgroup = {0};
memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); memcpy(&newVgroup, pVgroup, sizeof(SVgObj));
@ -744,9 +721,9 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj
return -1; return -1;
} }
newVgroup.replica = pDb->cfg.replications; newVgroup.replica = pDb->cfg.replications;
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1;
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1;
} else { } else {
mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId);
@ -757,7 +734,7 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj
return -1; return -1;
} }
newVgroup.replica = pDb->cfg.replications; newVgroup.replica = pDb->cfg.replications;
if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1;
if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1;
} }

3
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -197,8 +197,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
goto CONN_OVER; goto CONN_OVER;
} }
if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) {
mError("user:%s, failed to auth while acquire user, input:%s saved:%s", pReq->conn.user, connReq.passwd, mError("user:%s, failed to auth while acquire user, input:%s", pReq->conn.user, connReq.passwd);
pUser->pass);
code = TSDB_CODE_RPC_AUTH_FAILURE; code = TSDB_CODE_RPC_AUTH_FAILURE;
goto CONN_OVER; goto CONN_OVER;
} }

12
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -206,7 +206,7 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
int32_t totalVgNum = pOutput->pSub->vgNum; int32_t totalVgNum = pOutput->pSub->vgNum;
mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); mInfo("mq rebalance: subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum);
// 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
@ -231,6 +231,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp, .pVgEp = pVgEp,
}; };
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("mq rebalance: remove vg %d from consumer %ld", pVgEp->vgId, consumerId);
} }
taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
// put into removed // put into removed
@ -250,6 +251,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp, .pVgEp = pVgEp,
}; };
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
mInfo("mq rebalance: remove vg %d from unassigned", pVgEp->vgId);
} }
} }
@ -263,6 +265,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
minVgCnt = totalVgNum / afterRebConsumerNum; minVgCnt = totalVgNum / afterRebConsumerNum;
imbConsumerNum = totalVgNum % afterRebConsumerNum; imbConsumerNum = totalVgNum % afterRebConsumerNum;
} }
mInfo("mq rebalance: %d consumer after rebalance, at least %d vg each, %d consumer has more vg", afterRebConsumerNum,
minVgCnt, imbConsumerNum);
// 4. first scan: remove consumer more than wanted, put to remove hash // 4. first scan: remove consumer more than wanted, put to remove hash
int32_t imbCnt = 0; int32_t imbCnt = 0;
@ -290,6 +294,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp, .pVgEp = pVgEp,
}; };
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId);
} }
imbCnt++; imbCnt++;
} }
@ -303,6 +308,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp, .pVgEp = pVgEp,
}; };
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId);
} }
} }
} }
@ -319,6 +325,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
taosArrayPush(pOutput->newConsumers, &consumerId); taosArrayPush(pOutput->newConsumers, &consumerId);
mInfo("mq rebalance: add new consumer %ld", consumerId);
} }
} }
@ -343,6 +350,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pConsumerEp->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pOutput->rebVgs, pRebVg); taosArrayPush(pOutput->rebVgs, pRebVg);
mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
} }
} }
@ -360,6 +368,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pConsumerEp->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pOutput->rebVgs, pRebVg); taosArrayPush(pOutput->rebVgs, pRebVg);
mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
} }
} else { } else {
// if all consumer is removed, put all vg into unassigned // if all consumer is removed, put all vg into unassigned
@ -372,6 +381,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
ASSERT(pRebOutput->newConsumerId == -1); ASSERT(pRebOutput->newConsumerId == -1);
taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
taosArrayPush(pOutput->rebVgs, pRebOutput); taosArrayPush(pOutput->rebVgs, pRebOutput);
mInfo("mq rebalance: unassign vg %d (second scan)", pRebOutput->pVgEp->vgId);
} }
} }

14
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -842,13 +842,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
} }
taosMemoryFree(pTrans->rpcRsp); taosMemoryFree(pTrans->rpcRsp);
mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code, pTrans->stage, pTrans->rpcInfo.ahandle);
pTrans->rpcInfo.ahandle);
SRpcMsg rspMsg = { SRpcMsg rspMsg = {
.info = pTrans->rpcInfo,
.code = code, .code = code,
.pCont = rpcCont, .pCont = rpcCont,
.contLen = pTrans->rpcRspLen, .contLen = pTrans->rpcRspLen,
.info = pTrans->rpcInfo,
}; };
tmsgSendRsp(&rspMsg); tmsgSendRsp(&rspMsg);
pTrans->rpcInfo.handle = NULL; pTrans->rpcInfo.handle = NULL;
@ -899,7 +898,7 @@ void mndTransProcessRsp(SRpcMsg *pRsp) {
} }
} }
mDebug("trans:%d, action:%d response is received, code:0x%04x, accept:0x%04x", transId, action, pRsp->code, mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%04x", transId, action, pRsp->code,
pAction->acceptableCode); pAction->acceptableCode);
mndTransExecute(pMnode, pTrans); mndTransExecute(pMnode, pTrans);
@ -986,7 +985,8 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr
memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen);
if (tmsgSendReq(&pAction->epSet, &rpcMsg) == 0) { if (tmsgSendReq(&pAction->epSet, &rpcMsg) == 0) {
mDebug("trans:%d, action:%d is sent", pTrans->id, action); mDebug("trans:%d, action:%d is sent to %s:%u", pTrans->id, action, pAction->epSet.eps[pAction->epSet.inUse].fqdn,
pAction->epSet.eps[pAction->epSet.inUse].port);
pAction->msgSent = 1; pAction->msgSent = 1;
pAction->msgReceived = 0; pAction->msgReceived = 0;
pAction->errCode = 0; pAction->errCode = 0;
@ -1031,7 +1031,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions); mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions);
return 0; return 0;
} else { } else {
mError("trans:%d, all %d actions executed, code:0x%04x", pTrans->id, numOfActions, errCode & 0XFFFF); mError("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode & 0XFFFF);
mndTransResetActions(pMnode, pTrans, pArray); mndTransResetActions(pMnode, pTrans, pArray);
terrno = errCode; terrno = errCode;
return errCode; return errCode;
@ -1222,7 +1222,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr());
} }
mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); mDebug("trans:%d, finished, code:0x%x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes);
return continueExec; return continueExec;
} }

8
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -256,7 +256,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
return pReq; return pReq;
} }
void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) {
SAlterVnodeReq alterReq = {0}; SAlterVnodeReq alterReq = {0};
alterReq.vgVersion = pVgroup->version; alterReq.vgVersion = pVgroup->version;
alterReq.buffer = pDb->cfg.buffer; alterReq.buffer = pDb->cfg.buffer;
@ -285,16 +285,14 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgO
pReplica->port = pVgidDnode->port; pReplica->port = pVgidDnode->port;
memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN);
mndReleaseDnode(pMnode, pVgidDnode); mndReleaseDnode(pMnode, pVgidDnode);
if (pDnode->id == pVgid->dnodeId) {
alterReq.selfIndex = v;
}
} }
#if 0
if (alterReq.selfIndex == -1) { if (alterReq.selfIndex == -1) {
terrno = TSDB_CODE_MND_APP_ERROR; terrno = TSDB_CODE_MND_APP_ERROR;
return NULL; return NULL;
} }
#endif
int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq);
if (contLen < 0) { if (contLen < 0) {

9
source/dnode/mnode/impl/src/mnode.c
View File

@ -343,20 +343,21 @@ void mndStop(SMnode *pMnode) { return mndCleanupTimer(pMnode); }
int32_t mndProcessMsg(SRpcMsg *pMsg) { int32_t mndProcessMsg(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
void *ahandle = pMsg->info.ahandle; void *ahandle = pMsg->info.ahandle;
mTrace("msg:%p, will be processed, type:%s app:%p", pMsg, TMSG_INFO(pMsg->msgType), ahandle); mTrace("msg:%p, will be processed, type:%s app:%p", pMsg, TMSG_INFO(pMsg->msgType), ahandle);
if (IsReq(pMsg) && !mndIsMaster(pMnode)) { if (IsReq(pMsg)) {
if (!mndIsMaster(pMnode)) {
terrno = TSDB_CODE_APP_NOT_READY; terrno = TSDB_CODE_APP_NOT_READY;
mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
return -1; return -1;
} }
if (IsReq(pMsg) && (pMsg->contLen == 0 || pMsg->pCont == NULL)) { if (pMsg->contLen == 0 || pMsg->pCont == NULL) {
terrno = TSDB_CODE_INVALID_MSG_LEN; terrno = TSDB_CODE_INVALID_MSG_LEN;
mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
return -1; return -1;
} }
}
MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)]; MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)];
if (fp == NULL) { if (fp == NULL) {
@ -488,7 +489,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
tstrncpy(desc.status, "ready", sizeof(desc.status)); tstrncpy(desc.status, "ready", sizeof(desc.status));
pClusterInfo->vgroups_alive++; pClusterInfo->vgroups_alive++;
} }
if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_CANDIDATE) { if (pVgid->role != TAOS_SYNC_STATE_ERROR) {
pClusterInfo->vnodes_alive++; pClusterInfo->vnodes_alive++;
} }
pClusterInfo->vnodes_total++; pClusterInfo->vnodes_total++;

2
source/dnode/mnode/impl/test/trans/trans2.cpp
View File

@ -56,7 +56,7 @@ class MndTestTrans2 : public ::testing::Test {
msgCb.sendReqFp = sendReq; msgCb.sendReqFp = sendReq;
msgCb.sendRspFp = sendRsp; msgCb.sendRspFp = sendRsp;
msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack
tmsgSetDefaultMsgCb(&msgCb); tmsgSetDefault(&msgCb);
SMnodeOpt opt = {0}; SMnodeOpt opt = {0};
opt.deploy = 1; opt.deploy = 1;

2
source/dnode/vnode/inc/vnode.h
View File

@ -210,7 +210,7 @@ struct SMetaReader {
}; };
struct SMTbCursor { struct SMTbCursor {
TDBC *pDbc; TBC *pDbc;
void *pKey; void *pKey;
void *pVal; void *pVal;
int kLen; int kLen;

18
source/dnode/vnode/src/inc/meta.h
View File

@ -63,16 +63,16 @@ struct SMeta {
char* path; char* path;
SVnode* pVnode; SVnode* pVnode;
TENV* pEnv; TDB* pEnv;
TXN txn; TXN txn;
TDB* pTbDb; TTB* pTbDb;
TDB* pSkmDb; TTB* pSkmDb;
TDB* pUidIdx; TTB* pUidIdx;
TDB* pNameIdx; TTB* pNameIdx;
TDB* pCtbIdx; TTB* pCtbIdx;
TDB* pTagIdx; TTB* pTagIdx;
TDB* pTtlIdx; TTB* pTtlIdx;
TDB* pSmaIdx; TTB* pSmaIdx;
SMetaIdx* pIdx; SMetaIdx* pIdx;
}; };

10
source/dnode/vnode/src/inc/sma.h
View File

@ -46,7 +46,7 @@ struct SSmaEnv {
TXN txn; TXN txn;
void *pPool; // SPoolMem void *pPool; // SPoolMem
SDiskID did; SDiskID did;
TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? TDB *dbEnv; // TODO: If it's better to put it in smaIndex level?
char *path; // relative path char *path; // relative path
SSmaStat *pStat; SSmaStat *pStat;
}; };
@ -93,16 +93,16 @@ typedef struct SDBFile SDBFile;
struct SDBFile { struct SDBFile {
int32_t fid; int32_t fid;
TDB *pDB; TTB *pDB;
char *path; char *path;
}; };
int32_t tdSmaBeginCommit(SSmaEnv *pEnv); int32_t tdSmaBeginCommit(SSmaEnv *pEnv);
int32_t tdSmaEndCommit(SSmaEnv *pEnv); int32_t tdSmaEndCommit(SSmaEnv *pEnv);
int32_t smaOpenDBEnv(TENV **ppEnv, const char *path); int32_t smaOpenDBEnv(TDB **ppEnv, const char *path);
int32_t smaCloseDBEnv(TENV *pEnv); int32_t smaCloseDBEnv(TDB *pEnv);
int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF); int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF);
int32_t smaCloseDBF(SDBFile *pDBF); int32_t smaCloseDBF(SDBFile *pDBF);
int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);

1
source/dnode/vnode/src/inc/tq.h
View File

@ -179,6 +179,7 @@ struct STQ {
SHashObj* pStreamTasks; SHashObj* pStreamTasks;
SVnode* pVnode; SVnode* pVnode;
SWal* pWal; SWal* pWal;
// TDB* pTdb;
}; };
typedef struct { typedef struct {

1
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -117,6 +117,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
void tqClose(STQ*); void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqCommit(STQ*); int tqCommit(STQ*);
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList);
int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);

54
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -50,63 +50,63 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
taosMkDir(pMeta->path); taosMkDir(pMeta->path);
// open env // open env
ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pTbDb // open pTbDb
ret = tdbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pSkmDb // open pSkmDb
ret = tdbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pUidIdx // open pUidIdx
ret = tdbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pNameIdx // open pNameIdx
ret = tdbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pCtbIdx // open pCtbIdx
ret = tdbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pTagIdx // open pTagIdx
ret = tdbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pTtlIdx // open pTtlIdx
ret = tdbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pSmaIdx // open pSmaIdx
ret = tdbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
@ -125,15 +125,15 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
_err: _err:
if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); if (pMeta->pEnv) tdbClose(pMeta->pEnv);
metaDestroyLock(pMeta); metaDestroyLock(pMeta);
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
return -1; return -1;
@ -142,15 +142,15 @@ _err:
int metaClose(SMeta *pMeta) { int metaClose(SMeta *pMeta) {
if (pMeta) { if (pMeta) {
if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); if (pMeta->pEnv) tdbClose(pMeta->pEnv);
metaDestroyLock(pMeta); metaDestroyLock(pMeta);
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
} }

40
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -35,7 +35,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u
STbDbKey tbDbKey = {.version = version, .uid = uid}; STbDbKey tbDbKey = {.version = version, .uid = uid};
// query table.db // query table.db
if (tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
goto _err; goto _err;
} }
@ -58,7 +58,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
int64_t version; int64_t version;
// query uid.idx // query uid.idx
if (tdbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; return -1;
} }
@ -72,7 +72,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
tb_uid_t uid; tb_uid_t uid;
// query name.idx // query name.idx
if (tdbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { if (tdbTbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; return -1;
} }
@ -100,9 +100,9 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
metaReaderInit(&pTbCur->mr, pMeta, 0); metaReaderInit(&pTbCur->mr, pMeta, 0);
tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL);
tdbDbcMoveToFirst(pTbCur->pDbc); tdbTbcMoveToFirst(pTbCur->pDbc);
return pTbCur; return pTbCur;
} }
@ -113,7 +113,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) {
tdbFree(pTbCur->pVal); tdbFree(pTbCur->pVal);
metaReaderClear(&pTbCur->mr); metaReaderClear(&pTbCur->mr);
if (pTbCur->pDbc) { if (pTbCur->pDbc) {
tdbDbcClose(pTbCur->pDbc); tdbTbcClose(pTbCur->pDbc);
} }
taosMemoryFree(pTbCur); taosMemoryFree(pTbCur);
} }
@ -125,7 +125,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) {
STbCfg tbCfg; STbCfg tbCfg;
for (;;) { for (;;) {
ret = tdbDbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -159,7 +159,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
pKey = &skmDbKey; pKey = &skmDbKey;
kLen = sizeof(skmDbKey); kLen = sizeof(skmDbKey);
metaRLock(pMeta); metaRLock(pMeta);
ret = tdbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); ret = tdbTbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen);
metaULock(pMeta); metaULock(pMeta);
if (ret < 0) { if (ret < 0) {
return NULL; return NULL;
@ -184,7 +184,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
struct SMCtbCursor { struct SMCtbCursor {
SMeta *pMeta; SMeta *pMeta;
TDBC *pCur; TBC *pCur;
tb_uid_t suid; tb_uid_t suid;
void *pKey; void *pKey;
void *pVal; void *pVal;
@ -207,7 +207,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
pCtbCur->suid = uid; pCtbCur->suid = uid;
metaRLock(pMeta); metaRLock(pMeta);
ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); ret = tdbTbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL);
if (ret < 0) { if (ret < 0) {
metaULock(pMeta); metaULock(pMeta);
taosMemoryFree(pCtbCur); taosMemoryFree(pCtbCur);
@ -217,9 +217,9 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
// move to the suid // move to the suid
ctbIdxKey.suid = uid; ctbIdxKey.suid = uid;
ctbIdxKey.uid = INT64_MIN; ctbIdxKey.uid = INT64_MIN;
tdbDbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); tdbTbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c);
if (c > 0) { if (c > 0) {
tdbDbcMoveToNext(pCtbCur->pCur); tdbTbcMoveToNext(pCtbCur->pCur);
} }
return pCtbCur; return pCtbCur;
@ -229,7 +229,7 @@ void metaCloseCtbCursor(SMCtbCursor *pCtbCur) {
if (pCtbCur) { if (pCtbCur) {
if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta); if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta);
if (pCtbCur->pCur) { if (pCtbCur->pCur) {
tdbDbcClose(pCtbCur->pCur); tdbTbcClose(pCtbCur->pCur);
tdbFree(pCtbCur->pKey); tdbFree(pCtbCur->pKey);
tdbFree(pCtbCur->pVal); tdbFree(pCtbCur->pVal);
@ -243,7 +243,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
int ret; int ret;
SCtbIdxKey *pCtbIdxKey; SCtbIdxKey *pCtbIdxKey;
ret = tdbDbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); ret = tdbTbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen);
if (ret < 0) { if (ret < 0) {
return 0; return 0;
} }
@ -299,7 +299,7 @@ int metaGetTbNum(SMeta *pMeta) {
typedef struct { typedef struct {
SMeta *pMeta; SMeta *pMeta;
TDBC *pCur; TBC *pCur;
tb_uid_t uid; tb_uid_t uid;
void *pKey; void *pKey;
void *pVal; void *pVal;
@ -323,7 +323,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
pSmaCur->uid = uid; pSmaCur->uid = uid;
metaRLock(pMeta); metaRLock(pMeta);
ret = tdbDbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL); ret = tdbTbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL);
if (ret < 0) { if (ret < 0) {
metaULock(pMeta); metaULock(pMeta);
taosMemoryFree(pSmaCur); taosMemoryFree(pSmaCur);
@ -333,9 +333,9 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
// move to the suid // move to the suid
smaIdxKey.uid = uid; smaIdxKey.uid = uid;
smaIdxKey.smaUid = INT64_MIN; smaIdxKey.smaUid = INT64_MIN;
tdbDbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c); tdbTbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c);
if (c > 0) { if (c > 0) {
tdbDbcMoveToNext(pSmaCur->pCur); tdbTbcMoveToNext(pSmaCur->pCur);
} }
return pSmaCur; return pSmaCur;
@ -345,7 +345,7 @@ void metaCloseSmaCursor(SMSmaCursor *pSmaCur) {
if (pSmaCur) { if (pSmaCur) {
if (pSmaCur->pMeta) metaULock(pSmaCur->pMeta); if (pSmaCur->pMeta) metaULock(pSmaCur->pMeta);
if (pSmaCur->pCur) { if (pSmaCur->pCur) {
tdbDbcClose(pSmaCur->pCur); tdbTbcClose(pSmaCur->pCur);
tdbFree(pSmaCur->pKey); tdbFree(pSmaCur->pKey);
tdbFree(pSmaCur->pVal); tdbFree(pSmaCur->pVal);
@ -359,7 +359,7 @@ tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) {
int ret; int ret;
SSmaIdxKey *pSmaIdxKey; SSmaIdxKey *pSmaIdxKey;
ret = tdbDbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen); ret = tdbTbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen);
if (ret < 0) { if (ret < 0) {
return 0; return 0;
} }

8
source/dnode/vnode/src/meta/metaSma.c
View File

@ -117,7 +117,7 @@ static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
tEncoderClear(&coder); tEncoderClear(&coder);
// write to table.db // write to table.db
if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
goto _err; goto _err;
} }
@ -130,17 +130,17 @@ _err:
} }
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
} }
static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
} }
static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) {
SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid};
return tdbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn);
} }
static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) {

191
source/dnode/vnode/src/meta/metaTable.c
View File

@ -71,9 +71,9 @@ _err:
} }
int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
TDBC *pNameIdxc = NULL; TBC *pNameIdxc = NULL;
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TDBC *pCtbIdxc = NULL; TBC *pCtbIdxc = NULL;
SCtbIdxKey *pCtbIdxKey; SCtbIdxKey *pCtbIdxKey;
const void *pKey = NULL; const void *pKey = NULL;
int nKey; int nKey;
@ -82,43 +82,43 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
int c, ret; int c, ret;
// prepare uid idx cursor // prepare uid idx cursor
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
terrno = TSDB_CODE_VND_TB_NOT_EXIST; terrno = TSDB_CODE_VND_TB_NOT_EXIST;
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
goto _err; goto _err;
} }
// prepare name idx cursor // prepare name idx cursor
tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
ASSERT(0); ASSERT(0);
} }
tdbDbcDelete(pUidIdxc); tdbTbcDelete(pUidIdxc);
tdbDbcDelete(pNameIdxc); tdbTbcDelete(pNameIdxc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
tdbDbcClose(pNameIdxc); tdbTbcClose(pNameIdxc);
// loop to drop each child table // loop to drop each child table
tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c);
if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) { if (ret < 0 || (c < 0 && tdbTbcMoveToNext(pCtbIdxc) < 0)) {
tdbDbcClose(pCtbIdxc); tdbTbcClose(pCtbIdxc);
goto _exit; goto _exit;
} }
for (;;) { for (;;) {
tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); tdbTbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL);
pCtbIdxKey = (SCtbIdxKey *)pKey; pCtbIdxKey = (SCtbIdxKey *)pKey;
if (pCtbIdxKey->suid > pReq->suid) break; if (pCtbIdxKey->suid > pReq->suid) break;
// drop the child table (TODO) // drop the child table (TODO)
if (tdbDbcMoveToNext(pCtbIdxc) < 0) break; if (tdbTbcMoveToNext(pCtbIdxc) < 0) break;
} }
_exit: _exit:
@ -134,8 +134,8 @@ _err:
int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SMetaEntry oStbEntry = {0}; SMetaEntry oStbEntry = {0};
SMetaEntry nStbEntry = {0}; SMetaEntry nStbEntry = {0};
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TDBC *pTbDbc = NULL; TBC *pTbDbc = NULL;
const void *pData; const void *pData;
int nData; int nData;
int64_t oversion; int64_t oversion;
@ -143,14 +143,14 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
int32_t ret; int32_t ret;
int32_t c; int32_t c;
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c);
if (ret < 0 || c) { if (ret < 0 || c) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -158,11 +158,11 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
oversion = *(int64_t *)pData; oversion = *(int64_t *)pData;
tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
ret = tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c);
ASSERT(ret == 0 && c == 0); ASSERT(ret == 0 && c == 0);
ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
ASSERT(ret == 0); ASSERT(ret == 0);
tDecoderInit(&dc, pData, nData); tDecoderInit(&dc, pData, nData);
@ -191,12 +191,12 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
metaSaveToTbDb(pMeta, &nStbEntry); metaSaveToTbDb(pMeta, &nStbEntry);
// update uid index // update uid index
tdbDbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0);
metaULock(pMeta); metaULock(pMeta);
tDecoderClear(&dc); tDecoderClear(&dc);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return 0; return 0;
} }
@ -256,9 +256,9 @@ _err:
} }
int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
TDBC *pTbDbc = NULL; TBC *pTbDbc = NULL;
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TDBC *pNameIdxc = NULL; TBC *pNameIdxc = NULL;
const void *pData; const void *pData;
int nData; int nData;
tb_uid_t uid; tb_uid_t uid;
@ -271,15 +271,15 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
int c = 0, ret; int c = 0, ret;
// search & delete the name idx // search & delete the name idx
tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c);
if (ret < 0 || !tdbDbcIsValid(pNameIdxc) || c) { if (ret < 0 || !tdbTbcIsValid(pNameIdxc) || c) {
tdbDbcClose(pNameIdxc); tdbTbcClose(pNameIdxc);
terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
return -1; return -1;
} }
ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pNameIdxc, NULL, NULL, &pData, &nData);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -287,36 +287,36 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
uid = *(tb_uid_t *)pData; uid = *(tb_uid_t *)pData;
tdbDbcDelete(pNameIdxc); tdbTbcDelete(pNameIdxc);
tdbDbcClose(pNameIdxc); tdbTbcClose(pNameIdxc);
// search & delete uid idx // search & delete uid idx
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
tver = *(int64_t *)pData; tver = *(int64_t *)pData;
tdbDbcDelete(pUidIdxc); tdbTbcDelete(pUidIdxc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
// search and get meta entry // search and get meta entry
tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); ret = tdbTbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -345,21 +345,21 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
taosMemoryFree(pDataCopy); taosMemoryFree(pDataCopy);
tDecoderClear(&coder); tDecoderClear(&coder);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
if (type == TSDB_CHILD_TABLE) { if (type == TSDB_CHILD_TABLE) {
// remove the pCtbIdx // remove the pCtbIdx
TDBC *pCtbIdxc = NULL; TBC *pCtbIdxc = NULL;
tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
tdbDbcDelete(pCtbIdxc); tdbTbcDelete(pCtbIdxc);
tdbDbcClose(pCtbIdxc); tdbTbcClose(pCtbIdxc);
// remove tags from pTagIdx (todo) // remove tags from pTagIdx (todo)
} else if (type == TSDB_NORMAL_TABLE) { } else if (type == TSDB_NORMAL_TABLE) {
@ -389,7 +389,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
int c; int c;
// search name index // search name index
ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
if (ret < 0) { if (ret < 0) {
terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
return -1; return -1;
@ -400,22 +400,22 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
pVal = NULL; pVal = NULL;
// search uid index // search uid index
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
ASSERT(c == 0); ASSERT(c == 0);
tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
oversion = *(int64_t *)pData; oversion = *(int64_t *)pData;
// search table.db // search table.db
TDBC *pTbDbc = NULL; TBC *pTbDbc = NULL;
tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
ASSERT(c == 0); ASSERT(c == 0);
tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
// get table entry // get table entry
SDecoder dc = {0}; SDecoder dc = {0};
@ -443,6 +443,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
entry.version = version; entry.version = version;
int tlen; int tlen;
SSchema *pNewSchema = NULL;
switch (pAlterTbReq->action) { switch (pAlterTbReq->action) {
case TSDB_ALTER_TABLE_ADD_COLUMN: case TSDB_ALTER_TABLE_ADD_COLUMN:
if (pColumn) { if (pColumn) {
@ -451,8 +452,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
} }
pSchema->sver++; pSchema->sver++;
pSchema->nCols++; pSchema->nCols++;
pSchema->pSchema = pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols);
taosMemoryRealloc(entry.ntbEntry.schema.pSchema, sizeof(SSchema) * entry.ntbEntry.schema.nCols); memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1));
pSchema->pSchema = pNewSchema;
pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].bytes = pAlterTbReq->bytes; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].bytes = pAlterTbReq->bytes;
pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].type = pAlterTbReq->type; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].type = pAlterTbReq->type;
pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].flags = pAlterTbReq->flags; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].flags = pAlterTbReq->flags;
@ -505,21 +507,22 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
// save to table db // save to table db
metaSaveToTbDb(pMeta, &entry); metaSaveToTbDb(pMeta, &entry);
tdbDbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0); tdbTbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0);
metaSaveToSkmDb(pMeta, &entry); metaSaveToSkmDb(pMeta, &entry);
metaULock(pMeta); metaULock(pMeta);
if (pNewSchema) taosMemoryFree(pNewSchema);
tDecoderClear(&dc); tDecoderClear(&dc);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return 0; return 0;
_err: _err:
tDecoderClear(&dc); tDecoderClear(&dc);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return -1; return -1;
} }
@ -536,7 +539,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
int nData = 0; int nData = 0;
// search name index // search name index
ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
if (ret < 0) { if (ret < 0) {
terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
return -1; return -1;
@ -547,24 +550,24 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
pVal = NULL; pVal = NULL;
// search uid index // search uid index
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
ASSERT(c == 0); ASSERT(c == 0);
tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
oversion = *(int64_t *)pData; oversion = *(int64_t *)pData;
// search table.db // search table.db
TDBC *pTbDbc = NULL; TBC *pTbDbc = NULL;
SDecoder dc = {0}; SDecoder dc = {0};
/* get ctbEntry */ /* get ctbEntry */
tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
ASSERT(c == 0); ASSERT(c == 0);
tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
ctbEntry.pBuf = taosMemoryMalloc(nData); ctbEntry.pBuf = taosMemoryMalloc(nData);
memcpy(ctbEntry.pBuf, pData, nData); memcpy(ctbEntry.pBuf, pData, nData);
@ -573,8 +576,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
tDecoderClear(&dc); tDecoderClear(&dc);
/* get stbEntry*/ /* get stbEntry*/
tdbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal);
tdbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey),
(void **)&stbEntry.pBuf, &nVal); (void **)&stbEntry.pBuf, &nVal);
tdbFree(pVal); tdbFree(pVal);
tDecoderInit(&dc, stbEntry.pBuf, nVal); tDecoderInit(&dc, stbEntry.pBuf, nVal);
@ -632,19 +635,19 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
metaSaveToTbDb(pMeta, &ctbEntry); metaSaveToTbDb(pMeta, &ctbEntry);
// save to uid.idx // save to uid.idx
tdbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); tdbTbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn);
if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf);
if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return 0; return 0;
_err: _err:
if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf);
if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return -1; return -1;
} }
@ -708,7 +711,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
tEncoderClear(&coder); tEncoderClear(&coder);
// write to table.db // write to table.db
if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
goto _err; goto _err;
} }
@ -721,11 +724,11 @@ _err:
} }
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
} }
static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
} }
static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
@ -748,12 +751,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60;
ttlKey.uid = pME->uid; ttlKey.uid = pME->uid;
return tdbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
} }
static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
return tdbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
} }
static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid,
@ -801,10 +804,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
SDecoder dc = {0}; SDecoder dc = {0};
// get super table // get super table
tdbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData);
tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.uid = pCtbEntry->ctbEntry.suid;
tbDbKey.version = *(int64_t *)pData; tbDbKey.version = *(int64_t *)pData;
tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData);
tDecoderInit(&dc, pData, nData); tDecoderInit(&dc, pData, nData);
metaDecodeEntry(&dc, &stbEntry); metaDecodeEntry(&dc, &stbEntry);
@ -817,7 +820,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
&pTagIdxKey, &nTagIdxKey) < 0) { &pTagIdxKey, &nTagIdxKey) < 0) {
return -1; return -1;
} }
tdbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn);
metaDestroyTagIdxKey(pTagIdxKey); metaDestroyTagIdxKey(pTagIdxKey);
tDecoderClear(&dc); tDecoderClear(&dc);
@ -859,7 +862,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
tEncoderInit(&coder, pVal, vLen); tEncoderInit(&coder, pVal, vLen);
tEncodeSSchemaWrapper(&coder, pSW); tEncodeSSchemaWrapper(&coder, pSW);
if (tdbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) {
rcode = -1; rcode = -1;
goto _exit; goto _exit;
} }

18
source/dnode/vnode/src/sma/smaTDBImpl.c
View File

@ -17,12 +17,12 @@
#include "sma.h" #include "sma.h"
int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { int32_t smaOpenDBEnv(TDB **ppEnv, const char *path) {
int ret = 0; int ret = 0;
if (path == NULL) return -1; if (path == NULL) return -1;
ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param ret = tdbOpen(path, 4096, 256, ppEnv); // use as param
if (ret != 0) { if (ret != 0) {
smaError("failed to create tsdb db env, ret = %d", ret); smaError("failed to create tsdb db env, ret = %d", ret);
@ -32,7 +32,7 @@ int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) {
return 0; return 0;
} }
int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } int32_t smaCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); }
static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
const SSmaKey *pKey1 = (const SSmaKey *)arg1; const SSmaKey *pKey1 = (const SSmaKey *)arg1;
@ -54,21 +54,21 @@ static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int
return 0; return 0;
} }
static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { static int32_t smaOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) {
tdb_cmpr_fn_t compFunc; tdb_cmpr_fn_t compFunc;
// Create a database // Create a database
compFunc = tdSmaKeyCmpr; compFunc = tdSmaKeyCmpr;
if (tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { if (tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) {
return -1; return -1;
} }
return 0; return 0;
} }
static int32_t smaCloseDBDb(TDB *pDB) { return tdbClose(pDB); } static int32_t smaCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); }
int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) { int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF) {
// TEnv is shared by a group of SDBFile // TEnv is shared by a group of SDBFile
if (!pEnv || !pDBF) { if (!pEnv || !pDBF) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
@ -99,7 +99,7 @@ int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, in
int32_t ret; int32_t ret;
printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn); printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn);
ret = tdbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); ret = tdbTbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
if (ret < 0) { if (ret < 0) {
smaError("failed to upsert tsma data into db, ret = %d", ret); smaError("failed to upsert tsma data into db, ret = %d", ret);
return -1; return -1;
@ -112,7 +112,7 @@ void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_
void *pVal = NULL; void *pVal = NULL;
int ret; int ret;
ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
if (ret < 0) { if (ret < 0) {
smaError("failed to get tsma data from db, ret = %d", ret); smaError("failed to get tsma data from db, ret = %d", ret);

34
source/dnode/vnode/src/sma/smaTimeRange.c
View File

@ -55,7 +55,6 @@ typedef enum {
SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma
} ESmaStorageLevel; } ESmaStorageLevel;
// static func // static func
static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted);
@ -72,15 +71,12 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL
TXN *txn); TXN *txn);
// expired window // expired window
static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version);
static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey,
int64_t version);
static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey);
static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
// read data // read data
// implementation // implementation
/** /**
@ -157,7 +153,6 @@ static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
return false; return false;
} }
/** /**
* @brief Approximate value for week/month/year. * @brief Approximate value for week/month/year.
* *
@ -239,7 +234,6 @@ static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit,
return interval; return interval;
} }
static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
int8_t intervalUnit) { int8_t intervalUnit) {
pSmaH->pSma = pSma; pSmaH->pSma = pSma;
@ -493,7 +487,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
smaCloseDBF(&tSmaH.dFile); smaCloseDBF(&tSmaH.dFile);
} }
tdSetTSmaDataFile(&tSmaH, indexUid, fid); tdSetTSmaDataFile(&tSmaH, indexUid, fid);
smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 " queryKey:%" PRIi64, smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32
" queryKey:%" PRIi64,
SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey);
if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) {
smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma),
@ -525,7 +520,6 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64,
SMA_VID(pSma), skey, tlen, indexUid); SMA_VID(pSma), skey, tlen, indexUid);
} }
} }
} }
tdSmaEndCommit(pEnv); // TODO: not commit for every insert tdSmaEndCommit(pEnv); // TODO: not commit for every insert
@ -557,7 +551,7 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL
TXN *txn) { TXN *txn) {
SDBFile *pDBFile = &pSmaH->dFile; SDBFile *pDBFile = &pSmaH->dFile;
// TODO: insert tsma data blocks into B+Tree(TDB) // TODO: insert tsma data blocks into B+Tree(TTB)
if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
@ -600,8 +594,8 @@ static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUi
if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
// error handling // error handling
tdUnRefSmaStat(pSma, pStat); tdUnRefSmaStat(pSma, pStat);
smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey,
skey, indexUid); indexUid);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma),
@ -666,8 +660,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) {
smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal);
taosSsleep(1); taosSsleep(1);
if (++nSleep > SMA_DROP_EXPIRED_TIME) { if (++nSleep > SMA_DROP_EXPIRED_TIME) {
smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal);
refVal);
break; break;
}; };
} }
@ -736,8 +729,8 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) {
// TODO: mark this window as expired. // TODO: mark this window as expired.
smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey,
querySKey, indexUid); indexUid);
} else { } else {
smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey,
indexUid); indexUid);
@ -766,8 +759,8 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
int64_t queryGroupId = 0; int64_t queryGroupId = 0;
tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey);
smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path,
tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN);
void *result = NULL; void *result = NULL;
int32_t valueSize = 0; int32_t valueSize = 0;
@ -777,7 +770,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
smaCloseDBF(&tReadH.dFile); smaCloseDBF(&tReadH.dFile);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
#endif #endif
#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ #ifdef _TEST_SMA_PRINT_DEBUG_LOG_
for (uint32_t v = 0; v < valueSize; v += 8) { for (uint32_t v = 0; v < valueSize; v += 8) {
@ -932,8 +925,6 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
/** /**
* @brief Update expired window according to msg from stream computing module. * @brief Update expired window according to msg from stream computing module.
* *
@ -1044,4 +1035,3 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

19
source/dnode/vnode/src/tq/tq.c
View File

@ -32,6 +32,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
pTq->path = strdup(path); pTq->path = strdup(path);
pTq->pVnode = pVnode; pTq->pVnode = pVnode;
pTq->pWal = pWal; pTq->pWal = pWal;
/*if (tdbOpen(path, 4096, 1, &pTq->pTdb) < 0) {*/
/*ASSERT(0);*/
/*}*/
#if 0 #if 0
pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer,
@ -101,6 +104,22 @@ static void tdSRowDemo() {
taosMemoryFree(pTSChema); taosMemoryFree(pTSChema);
} }
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList) {
void* pIter = NULL;
STqExec* pExec = NULL;
while (1) {
pIter = taosHashIterate(pTq->execs, pIter);
if (pIter == NULL) break;
pExec = (STqExec*)pIter;
if (pExec->subType == TOPIC_SUB_TYPE__DB) continue;
for (int32_t i = 0; i < 5; i++) {
int32_t code = qUpdateQualifiedTableId(pExec->task[i], tbUidList, true);
ASSERT(code == 0);
}
}
return 0;
}
int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
if (msgType != TDMT_VND_SUBMIT) return 0; if (msgType != TDMT_VND_SUBMIT) return 0;
void* pIter = NULL; void* pIter = NULL;

4
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
View File

@ -330,12 +330,12 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
ASSERT(pReadh->pDCols[0]->bitmapMode != 0); ASSERT(pReadh->pDCols[0]->bitmapMode != 0);
} }
if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) {
for (int i = 0; i < numOfColsIds; ++i) { for (int i = 0; i < numOfColsIds; ++i) {
SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; SDataCol *pDataCol = pReadh->pDCols[0]->cols + i;
if (pDataCol->bitmap) { if (pDataCol->len > 0 && pDataCol->bitmap) {
ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID); ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID);
ASSERT(pDataCol->pBitmap);
tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap); tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap);
tdDataColsSetBitmapI(pReadh->pDCols[0]); tdDataColsSetBitmapI(pReadh->pDCols[0]);
} }

4
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -53,7 +53,7 @@ struct SSmaEnv {
TXN txn; TXN txn;
SPoolMem *pPool; SPoolMem *pPool;
SDiskID did; SDiskID did;
TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? TDB *dbEnv; // TODO: If it's better to put it in smaIndex level?
char *path; // relative path char *path; // relative path
SSmaStat *pStat; SSmaStat *pStat;
}; };
@ -876,7 +876,7 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t ke
TXN *txn) { TXN *txn) {
SDBFile *pDBFile = &pSmaH->dFile; SDBFile *pDBFile = &pSmaH->dFile;
// TODO: insert tsma data blocks into B+Tree(TDB) // TODO: insert tsma data blocks into B+Tree(TTB)
if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);

18
source/dnode/vnode/src/tsdb/tsdbTDBImpl.c
View File

@ -17,12 +17,12 @@
#include "tsdb.h" #include "tsdb.h"
int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { int32_t tsdbOpenDBEnv(TDB **ppEnv, const char *path) {
int ret = 0; int ret = 0;
if (path == NULL) return -1; if (path == NULL) return -1;
ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param ret = tdbOpen(path, 4096, 256, ppEnv); // use as param
if (ret != 0) { if (ret != 0) {
tsdbError("Failed to create tsdb db env, ret = %d", ret); tsdbError("Failed to create tsdb db env, ret = %d", ret);
@ -32,7 +32,7 @@ int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) {
return 0; return 0;
} }
int32_t tsdbCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } int32_t tsdbCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); }
static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
const SSmaKey *pKey1 = (const SSmaKey *)arg1; const SSmaKey *pKey1 = (const SSmaKey *)arg1;
@ -54,20 +54,20 @@ static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, i
return 0; return 0;
} }
static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { static int32_t tsdbOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) {
int ret; int ret;
tdb_cmpr_fn_t compFunc; tdb_cmpr_fn_t compFunc;
// Create a database // Create a database
compFunc = tsdbSmaKeyCmpr; compFunc = tsdbSmaKeyCmpr;
ret = tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); ret = tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB);
return 0; return 0;
} }
static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbClose(pDB); } static int32_t tsdbCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); }
int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF) { int32_t tsdbOpenDBF(TDB *pEnv, SDBFile *pDBF) {
// TEnv is shared by a group of SDBFile // TEnv is shared by a group of SDBFile
if (!pEnv || !pDBF) { if (!pEnv || !pDBF) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) {
int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
int32_t ret; int32_t ret;
ret = tdbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); ret = tdbTbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
if (ret < 0) { if (ret < 0) {
tsdbError("Failed to create insert sma data into db, ret = %d", ret); tsdbError("Failed to create insert sma data into db, ret = %d", ret);
return -1; return -1;
@ -110,7 +110,7 @@ void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32
void *pVal = NULL; void *pVal = NULL;
int ret; int ret;
ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
if (ret < 0) { if (ret < 0) {
tsdbError("Failed to get sma data from db, ret = %d", ret); tsdbError("Failed to get sma data from db, ret = %d", ret);

7
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -333,6 +333,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
SVCreateTbRsp cRsp = {0}; SVCreateTbRsp cRsp = {0};
char tbName[TSDB_TABLE_FNAME_LEN]; char tbName[TSDB_TABLE_FNAME_LEN];
STbUidStore *pStore = NULL; STbUidStore *pStore = NULL;
SArray *tbUids = NULL;
pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
pRsp->code = TSDB_CODE_SUCCESS; pRsp->code = TSDB_CODE_SUCCESS;
@ -348,7 +349,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
} }
rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp));
if (rsp.pArray == NULL) { tbUids = taosArrayInit(req.nReqs, sizeof(int64_t));
if (rsp.pArray == NULL || tbUids == NULL) {
rcode = -1; rcode = -1;
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
@ -376,6 +378,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
} else { } else {
cRsp.code = TSDB_CODE_SUCCESS; cRsp.code = TSDB_CODE_SUCCESS;
tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
taosArrayPush(tbUids, &pCreateReq->uid);
} }
taosArrayPush(rsp.pArray, &cRsp); taosArrayPush(rsp.pArray, &cRsp);
@ -383,6 +386,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
tDecoderClear(&decoder); tDecoderClear(&decoder);
tqUpdateTbUidList(pVnode->pTq, tbUids);
tdUpdateTbUidList(pVnode->pSma, pStore); tdUpdateTbUidList(pVnode->pSma, pStore);
tdUidStoreFree(pStore); tdUidStoreFree(pStore);
@ -402,6 +406,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
_exit: _exit:
taosArrayDestroy(rsp.pArray); taosArrayDestroy(rsp.pArray);
taosArrayDestroy(tbUids);
tDecoderClear(&decoder); tDecoderClear(&decoder);
tEncoderClear(&encoder); tEncoderClear(&encoder);
return rcode; return rcode;

27
source/libs/executor/inc/executorimpl.h
View File

@ -378,6 +378,13 @@ typedef enum EStreamScanMode {
STREAM_SCAN_FROM_DATAREADER, STREAM_SCAN_FROM_DATAREADER,
} EStreamScanMode; } EStreamScanMode;
typedef struct SCatchSupporter {
SHashObj* pWindowHashTable; // quick locate the window object for each window
SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file
int32_t keySize;
int64_t* pKeyBuf;
} SCatchSupporter;
typedef struct SStreamBlockScanInfo { typedef struct SStreamBlockScanInfo {
SArray* pBlockLists; // multiple SSDatablock. SArray* pBlockLists; // multiple SSDatablock.
SSDataBlock* pRes; // result SSDataBlock SSDataBlock* pRes; // result SSDataBlock
@ -400,6 +407,8 @@ typedef struct SStreamBlockScanInfo {
EStreamScanMode scanMode; EStreamScanMode scanMode;
SOperatorInfo* pOperatorDumy; SOperatorInfo* pOperatorDumy;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
SCatchSupporter childAggSup;
SArray* childIds;
} SStreamBlockScanInfo; } SStreamBlockScanInfo;
typedef struct SSysTableScanInfo { typedef struct SSysTableScanInfo {
@ -460,6 +469,16 @@ typedef struct SIntervalAggOperatorInfo {
bool invertible; bool invertible;
} SIntervalAggOperatorInfo; } SIntervalAggOperatorInfo;
typedef struct SStreamFinalIntervalOperatorInfo {
SOptrBasicInfo binfo; // basic info
SGroupResInfo groupResInfo; // multiple results build supporter
SInterval interval; // interval info
int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator.
SAggSupporter aggSup; // aggregate supporter
int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup;
} SStreamFinalIntervalOperatorInfo;
typedef struct SAggOperatorInfo { typedef struct SAggOperatorInfo {
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SAggSupporter aggSup; SAggSupporter aggSup;
@ -696,6 +715,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
@ -771,9 +793,8 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary
TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item,
int32_t order); int32_t order);
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize,
void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, const char* pKey, const char* pDir);
uint64_t groupId, int32_t numOfOutput);
#ifdef __cplusplus #ifdef __cplusplus
} }

10
source/libs/executor/src/executor.c
View File

@ -106,7 +106,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
pMsg->contentLen = pMsg->contentLen; pMsg->contentLen = pMsg->contentLen;
#endif #endif
qDebugL("stream task string %s", (const char*)msg); /*qDebugL("stream task string %s", (const char*)msg);*/
struct SSubplan* plan = NULL; struct SSubplan* plan = NULL;
int32_t code = qStringToSubplan(msg, &plan); int32_t code = qStringToSubplan(msg, &plan);
@ -126,7 +126,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
return pTaskInfo; return pTaskInfo;
} }
int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
// traverse to the stream scanner node to add this table id // traverse to the stream scanner node to add this table id
@ -141,12 +141,12 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); metaReaderInit(&mr, pScanInfo->readHandle.meta, 0);
for(int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) {
int64_t* id = (int64_t*)taosArrayGet(tableIdList, i); int64_t* id = (int64_t*)taosArrayGet(tableIdList, i);
int32_t code = metaGetTableEntryByUid(&mr, *id); int32_t code = metaGetTableEntryByUid(&mr, *id);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:%"PRIu64" code:%s", *id, tstrerror(terrno)); qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
continue; continue;
} }
@ -160,7 +160,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
metaReaderClear(&mr); metaReaderClear(&mr);
qDebug(" %d qualified child tables added into stream scanner", (int32_t) taosArrayGetSize(qa)); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa); int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;

41
source/libs/executor/src/executorimpl.c
View File

@ -342,28 +342,6 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId,
return pResultRow; return pResultRow;
} }
void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes,
uint64_t groupId, int32_t numOfOutput) {
SAggSupporter* pSup = &pInfo->aggSup;
SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
SResultRowPosition* p1 =
(SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
GET_RES_WINDOW_KEY_LEN(bytes));
SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1);
SqlFunctionCtx* pCtx = pInfo->binfo.pCtx;
for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].resultInfo = getResultCell(pResult, i, pInfo->binfo.rowCellInfoOffset);
struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
continue;
}
pResInfo->initialized = false;
if (pCtx[i].functionId != -1) {
pCtx[i].fpSet.init(&pCtx[i], pResInfo);
}
}
}
/** /**
* the struct of key in hash table * the struct of key in hash table
* +----------+---------------+ * +----------+---------------+
@ -891,7 +869,11 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SColumnInfoData idata = {.info = pResColData->info, .hasNull = true};
SScalarParam dest = {.columnData = &idata}; SScalarParam dest = {.columnData = &idata};
scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pBlockList);
return code;
}
int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; int32_t startOffset = createNewColModel ? 0 : pResult->info.rows;
colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity); colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity);
@ -5321,3 +5303,16 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize,
const char* pKey, const char* pDir) {
pCatchSup->keySize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY);
pCatchSup->pKeyBuf = taosMemoryCalloc(1, pCatchSup->keySize);
int32_t pageSize = rowSize * 32;
int32_t bufSize = pageSize * 4096;
createDiskbasedBuf(&pCatchSup->pDataBuf, pageSize, bufSize, pKey, pDir);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK);;
return TSDB_CODE_SUCCESS;
}

122
source/libs/executor/src/scanoperator.c
View File

@ -36,6 +36,11 @@
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
typedef struct SWindowPosition {
int32_t pageId;
int32_t rowId;
} SWindowPosition;
static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity);
static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size,
const char* dbName); const char* dbName);
@ -668,12 +673,103 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool inverti
} }
pDataBlock->info.rows = size; pDataBlock->info.rows = size;
pDataBlock->info.type = STREAM_REPROCESS; pDataBlock->info.type = STREAM_REPROCESS;
blockDataUpdateTsWindow(pDataBlock);
taosArrayClear(pInfo->tsArray); taosArrayClear(pInfo->tsArray);
return pDataBlock; return pDataBlock;
} }
return NULL; return NULL;
} }
void static setSupKeyBuf(SCatchSupporter* pSup, int64_t groupId, int64_t childId, TSKEY ts) {
int64_t* pKey = (int64_t*)pSup->pKeyBuf;
pKey[0] = groupId;
pKey[1] = childId;
pKey[2] = ts;
}
static int32_t catchWidonwInfo(SSDataBlock* pDataBlock, SCatchSupporter* pSup,
int32_t pageId, int32_t tsIndex, int64_t childId) {
SColumnInfoData* pColDataInfo = taosArrayGet(pDataBlock->pDataBlock, tsIndex);
TSKEY* tsCols = (int64_t*)pColDataInfo->pData;
for (int32_t i = 0; i < pDataBlock->info.rows; i++) {
setSupKeyBuf(pSup, pDataBlock->info.groupId, childId, tsCols[i]);
SWindowPosition* p1 = (SWindowPosition*)taosHashGet(pSup->pWindowHashTable,
pSup->pKeyBuf, pSup->keySize);
if (p1 == NULL) {
SWindowPosition pos = {.pageId = pageId, .rowId = i};
int32_t code = taosHashPut(pSup->pWindowHashTable, pSup->pKeyBuf, pSup->keySize, &pos,
sizeof(SWindowPosition));
if (code != TSDB_CODE_SUCCESS ) {
return code;
}
} else {
p1->pageId = pageId;
p1->rowId = i;
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t catchDatablock(SSDataBlock* pDataBlock, SCatchSupporter* pSup,
int32_t tsIndex, int64_t childId) {
int32_t start = 0;
int32_t stop = 0;
int32_t pageSize = getBufPageSize(pSup->pDataBuf);
while(start < pDataBlock->info.rows) {
blockDataSplitRows(pDataBlock, pDataBlock->info.hasVarCol, start, &stop, pageSize);
SSDataBlock* pDB = blockDataExtractBlock(pDataBlock, start, stop - start + 1);
if (pDB == NULL) {
return terrno;
}
int32_t pageId = -1;
void* pPage = getNewBufPage(pSup->pDataBuf, pDataBlock->info.groupId, &pageId);
if (pPage == NULL) {
blockDataDestroy(pDB);
return terrno;
}
int32_t size = blockDataGetSize(pDB) + sizeof(int32_t) + pDB->info.numOfCols * sizeof(int32_t);
assert(size <= pageSize);
blockDataToBuf(pPage, pDB);
setBufPageDirty(pPage, true);
releaseBufPage(pSup->pDataBuf, pPage);
blockDataDestroy(pDB);
start = stop + 1;
int32_t code = catchWidonwInfo(pDB, pSup, pageId, tsIndex, childId);
if (code != TSDB_CODE_SUCCESS ) {
return code;
}
}
return TSDB_CODE_SUCCESS;
}
static SSDataBlock* getDataFromCatch(SStreamBlockScanInfo* pInfo) {
SSDataBlock* pBlock = pInfo->pUpdateRes;
if (pInfo->updateResIndex < pBlock->info.rows) {
blockDataCleanup(pInfo->pRes);
SCatchSupporter* pCSup = &pInfo->childAggSup;
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, 0);
TSKEY *tsCols = (TSKEY*)pColDataInfo->pData;
int32_t size = taosArrayGetSize(pInfo->childIds);
for (int32_t i = 0; i < size; i++) {
int64_t id = *(int64_t *)taosArrayGet(pInfo->childIds, i);
setSupKeyBuf(pCSup, pBlock->info.groupId, id,
tsCols[pInfo->updateResIndex]);
SWindowPosition* pos = (SWindowPosition*)taosHashGet(pCSup->pWindowHashTable,
pCSup->pKeyBuf, pCSup->keySize);
void* buf = getBufPage(pCSup->pDataBuf, pos->pageId);
SSDataBlock* pDB = createOneDataBlock(pInfo->pRes, false);
blockDataFromBuf(pDB, buf);
SSDataBlock* pSub = blockDataExtractBlock(pDB, pos->rowId, 1);
blockDataMerge(pInfo->pRes, pSub, NULL);
blockDataDestroy(pDB);
blockDataDestroy(pSub);
}
pInfo->updateResIndex++;
return pInfo->pRes;
}
return NULL;
}
static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not // NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -687,6 +783,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
size_t total = taosArrayGetSize(pInfo->pBlockLists); size_t total = taosArrayGetSize(pInfo->pBlockLists);
if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) {
SSDataBlock* pDB = getDataFromCatch(pInfo);
if (pDB != NULL) {
return pDB;
} else {
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
}
}
if (pInfo->validBlockIndex >= total) { if (pInfo->validBlockIndex >= total) {
doClearBufferedBlocks(pInfo); doClearBufferedBlocks(pInfo);
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
@ -694,7 +799,17 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
} }
int32_t current = pInfo->validBlockIndex++; int32_t current = pInfo->validBlockIndex++;
return taosArrayGetP(pInfo->pBlockLists, current); SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
if (pBlock->info.type == STREAM_REPROCESS) {
pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
} else {
int32_t code = catchDatablock(pBlock, &pInfo->childAggSup, pInfo->primaryTsIndex, 0);
if (code != TDB_CODE_SUCCESS) {
pTaskInfo->code = code;
longjmp(pTaskInfo->env, code);
}
}
return pBlock;
} else { } else {
if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
blockDataDestroy(pInfo->pUpdateRes); blockDataDestroy(pInfo->pUpdateRes);
@ -768,6 +883,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
} }
rows = pBlockInfo->rows; rows = pBlockInfo->rows;
doFilter(pInfo->pCondition, pInfo->pRes, NULL); doFilter(pInfo->pCondition, pInfo->pRes, NULL);
blockDataUpdateTsWindow(pInfo->pRes);
break; break;
} }
@ -855,6 +971,10 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR
pInfo->pOperatorDumy = pOperatorDumy; pInfo->pOperatorDumy = pOperatorDumy;
pInfo->interval = pSTInfo->interval; pInfo->interval = pSTInfo->interval;
size_t childKeyBufSize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY);
initCatchSupporter(&pInfo->childAggSup, 1024, childKeyBufSize,
"StreamFinalInterval", "/tmp/"); // TODO(liuyao) get row size from phy plan
pOperator->name = "StreamBlockScanOperator"; pOperator->name = "StreamBlockScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
pOperator->blocking = false; pOperator->blocking = false;

196
source/libs/executor/src/timewindowoperator.c
View File

@ -8,6 +8,8 @@ typedef enum SResultTsInterpType {
RESULT_ROW_END_INTERP = 2, RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType; } SResultTsInterpType;
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator);
/* /*
* There are two cases to handle: * There are two cases to handle:
* *
@ -473,8 +475,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun
} }
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
TSKEY* primaryKeys, int32_t prevPosition, SIntervalAggOperatorInfo* pInfo) { TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
int32_t order = pInfo->order;
bool ascQuery = (order == TSDB_ORDER_ASC); bool ascQuery = (order == TSDB_ORDER_ASC);
int32_t precision = pInterval->precision; int32_t precision = pInterval->precision;
@ -723,7 +724,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
STimeWindow nextWin = win; STimeWindow nextWin = win;
while (1) { while (1) {
int32_t prevEndPos = (forwardStep - 1) * step + startPos; int32_t prevEndPos = (forwardStep - 1) * step + startPos;
startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
if (startPos < 0) { if (startPos < 0) {
break; break;
} }
@ -1031,18 +1032,41 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type
} }
} }
} }
static void doClearWindows(SIntervalAggOperatorInfo* pInfo, int32_t numOfOutput, SSDataBlock* pBlock) {
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData,
int16_t bytes, uint64_t groupId, int32_t numOfOutput) {
SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
SResultRowPosition* p1 =
(SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
GET_RES_WINDOW_KEY_LEN(bytes));
SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1);
SqlFunctionCtx* pCtx = pBinfo->pCtx;
for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].resultInfo = getResultCell(pResult, i, pBinfo->rowCellInfoOffset);
struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
continue;
}
pResInfo->initialized = false;
if (pCtx[i].functionId != -1) {
pCtx[i].fpSet.init(&pCtx[i], pResInfo);
}
}
}
static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo,
SInterval* pIntrerval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock) {
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; TSKEY *tsCols = (TSKEY*)pColDataInfo->pData;
int32_t step = 0; int32_t step = 0;
for (int32_t i = 0; i < pBlock->info.rows; i += step) { for (int32_t i = 0; i < pBlock->info.rows; i += step) {
SResultRowInfo dumyInfo; SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1; dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], &pInfo->interval, STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pIntrerval,
pInfo->interval.precision, NULL); pIntrerval->precision, NULL);
step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i,
win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
doClearWindow(pInfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); doClearWindow(pSup, pBinfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput);
} }
} }
@ -1084,7 +1108,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type); setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type);
} }
if (pBlock->info.type == STREAM_REPROCESS) { if (pBlock->info.type == STREAM_REPROCESS) {
doClearWindows(pInfo, pOperator->numOfExprs, pBlock); doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval,
pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock);
continue; continue;
} }
pInfo->order = TSDB_ORDER_ASC; pInfo->order = TSDB_ORDER_ASC;
@ -1097,8 +1122,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
// TODO: remove for stream
/*ASSERT(pInfo->binfo.pRes->info.rows > 0);*/
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
@ -1116,6 +1139,12 @@ void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);
} }
void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo *)param;
doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
cleanupAggSup(&pInfo->aggSup);
}
bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
for (int32_t i = 0; i < numOfCols; i++) { for (int32_t i = 0; i < numOfCols; i++) {
if (!fmIsInvertible(pFCtx[i].functionId)) { if (!fmIsInvertible(pFCtx[i].functionId)) {
@ -1185,6 +1214,63 @@ _error:
return NULL; return NULL;
} }
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo,
SExecTaskInfo* pTaskInfo) {
SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
pInfo->order = TSDB_ORDER_ASC;
pInfo->interval = *pInterval;
pInfo->twAggSup = *pTwAggSupp;
pInfo->primaryTsIndex = primaryTsSlotId;
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
initResultSizeInfo(pOperator, 4096);
int32_t code =
initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock,
keyBufSize, pTaskInfo->id.str);
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1);
pOperator->name = "StreamFinalIntervalOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
pOperator->blocking = true;
pOperator->status = OP_NOT_OPENED;
pOperator->pExpr = pExprInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->numOfExprs = numOfCols;
pOperator->info = pInfo;
pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL,
destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow,
NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
return pOperator;
_error:
destroyStreamFinalIntervalOperatorInfo(pInfo, numOfCols);
taosMemoryFreeClear(pInfo);
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
return NULL;
}
SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo,
@ -1548,3 +1634,91 @@ _error:
pTaskInfo->code = code; pTaskInfo->code = code;
return NULL; return NULL;
} }
static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock,
int32_t tableGroupId) {
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
int32_t numOfOutput = pOperatorInfo->numOfExprs;
SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
int32_t step = 1;
bool ascScan = true;
TSKEY* tsCols = NULL;
SResultRow* pResult = NULL;
int32_t forwardStep = 0;
if (pSDataBlock->pDataBlock != NULL) {
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData;
}
int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan);
STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts,
&pInfo->interval, pInfo->interval.precision, NULL);
while (1) {
int32_t code =
setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx,
numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
pos->groupId = tableGroupId;
pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
*(int64_t*)pos->key = pResult->win.skey;
taosArrayPush(pUpdated, &pos);
forwardStep =
getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
// window start(end) key interpolation
doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep,
pInfo->order, false);
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols,
pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
int32_t prevEndPos = (forwardStep - 1) * step + startPos;
startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
if (startPos < 0) {
break;
}
}
return pUpdated;
}
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
SOperatorInfo* downstream = pOperator->pDownstream[0];
SArray* pUpdated = NULL;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
} else if (pOperator->status == OP_RES_TO_RETURN) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
pOperator->status = OP_EXEC_DONE;
}
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}
while (1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
break;
}
setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
if (pBlock->info.type == STREAM_REPROCESS) {
doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval,
pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock);
continue;
}
pUpdated = doHashInterval(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0);
}
finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
pOperator->status = OP_RES_TO_RETURN;
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

1
source/libs/function/inc/builtinsimpl.h
View File

@ -74,6 +74,7 @@ int32_t diffFunction(SqlFunctionCtx *pCtx);
bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t firstFunction(SqlFunctionCtx *pCtx);
int32_t lastFunction(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx);
int32_t lastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
int32_t topFunction(SqlFunctionCtx *pCtx); int32_t topFunction(SqlFunctionCtx *pCtx);

13
source/libs/function/src/builtins.c
View File

@ -47,8 +47,10 @@ static int32_t translateInOutNum(SFunctionNode* pFunc, char* pErrBuf, int32_t le
} }
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_NUMERIC_TYPE(paraType)) { if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
} else if (IS_NULL_TYPE(paraType)) {
paraType = TSDB_DATA_TYPE_BIGINT;
} }
pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType}; pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType};
@ -62,7 +64,7 @@ static int32_t translateInNumOutDou(SFunctionNode* pFunc, char* pErrBuf, int32_t
} }
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_NUMERIC_TYPE(paraType)) { if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
} }
@ -115,18 +117,19 @@ static int32_t translateSum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
} }
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_NUMERIC_TYPE(paraType)) { if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
} }
uint8_t resType = 0; uint8_t resType = 0;
if (IS_SIGNED_NUMERIC_TYPE(paraType) || paraType == TSDB_DATA_TYPE_BOOL) { if (IS_SIGNED_NUMERIC_TYPE(paraType) || TSDB_DATA_TYPE_BOOL == paraType || IS_NULL_TYPE(paraType)) {
resType = TSDB_DATA_TYPE_BIGINT; resType = TSDB_DATA_TYPE_BIGINT;
} else if (IS_UNSIGNED_NUMERIC_TYPE(paraType)) { } else if (IS_UNSIGNED_NUMERIC_TYPE(paraType)) {
resType = TSDB_DATA_TYPE_UBIGINT; resType = TSDB_DATA_TYPE_UBIGINT;
} else if (IS_FLOAT_TYPE(paraType)) { } else if (IS_FLOAT_TYPE(paraType)) {
resType = TSDB_DATA_TYPE_DOUBLE; resType = TSDB_DATA_TYPE_DOUBLE;
} }
pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType};
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -872,7 +875,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.getEnvFunc = getFirstLastFuncEnv, .getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup, .initFunc = functionSetup,
.processFunc = lastFunction, .processFunc = lastFunction,
.finalizeFunc = functionFinalize .finalizeFunc = lastFinalize
}, },
{ {
.name = "diff", .name = "diff",

57
source/libs/function/src/builtinsimpl.c
View File

@ -255,7 +255,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; //pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0;
char* in = GET_ROWCELL_INTERBUF(pResInfo); char* in = GET_ROWCELL_INTERBUF(pResInfo);
colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes);
@ -331,8 +331,18 @@ static FORCE_INLINE int32_t getNumofElem(SqlFunctionCtx* pCtx) {
int32_t countFunction(SqlFunctionCtx* pCtx) { int32_t countFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElem = getNumofElem(pCtx); int32_t numOfElem = getNumofElem(pCtx);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
char* buf = GET_ROWCELL_INTERBUF(pResInfo); char* buf = GET_ROWCELL_INTERBUF(pResInfo);
if (IS_NULL_TYPE(type)) {
//select count(NULL) returns 0
numOfElem = 1;
*((int64_t*)buf) = 0;
} else {
*((int64_t*)buf) += numOfElem; *((int64_t*)buf) += numOfElem;
}
SET_VAL(pResInfo, numOfElem, 1); SET_VAL(pResInfo, numOfElem, 1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -383,6 +393,12 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (IS_NULL_TYPE(type)) {
GET_RES_INFO(pCtx)->isNullRes = 1;
numOfElem = 1;
goto _sum_over;
}
if (pInput->colDataAggIsSet) { if (pInput->colDataAggIsSet) {
numOfElem = pInput->numOfRows - pAgg->numOfNull; numOfElem = pInput->numOfRows - pAgg->numOfNull;
ASSERT(numOfElem >= 0); ASSERT(numOfElem >= 0);
@ -427,6 +443,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
} }
} }
_sum_over:
// data in the check operation are all null, not output // data in the check operation are all null, not output
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -526,6 +543,12 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex; int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows; int32_t numOfRows = pInput->numOfRows;
if (IS_NULL_TYPE(type)) {
GET_RES_INFO(pCtx)->isNullRes = 1;
numOfElem = 1;
goto _avg_over;
}
switch (type) { switch (type) {
case TSDB_DATA_TYPE_TINYINT: { case TSDB_DATA_TYPE_TINYINT: {
int8_t* plist = (int8_t*)pCol->pData; int8_t* plist = (int8_t*)pCol->pData;
@ -617,6 +640,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
break; break;
} }
_avg_over:
// data in the check operation are all null, not output // data in the check operation are all null, not output
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -783,6 +807,12 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SMinmaxResInfo *pBuf = GET_ROWCELL_INTERBUF(pResInfo); SMinmaxResInfo *pBuf = GET_ROWCELL_INTERBUF(pResInfo);
if (IS_NULL_TYPE(type)) {
GET_RES_INFO(pCtx)->isNullRes = 1;
numOfElems = 1;
goto _min_max_over;
}
// data in current data block are qualified to the query // data in current data block are qualified to the query
if (pInput->colDataAggIsSet) { if (pInput->colDataAggIsSet) {
numOfElems = pInput->numOfRows - pAgg->numOfNull; numOfElems = pInput->numOfRows - pAgg->numOfNull;
@ -1183,6 +1213,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
} }
} }
_min_max_over:
return numOfElems; return numOfElems;
} }
@ -1215,9 +1246,9 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) {
float v = *(double*) &pRes->v; float v = *(double*) &pRes->v;
colDataAppend(pCol, currentRow, (const char*)&v, false); colDataAppend(pCol, currentRow, (const char*)&v, pEntryInfo->isNullRes);
} else { } else {
colDataAppend(pCol, currentRow, (const char*)&pRes->v, false); colDataAppend(pCol, currentRow, (const char*)&pRes->v, pEntryInfo->isNullRes);
} }
setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow);
@ -1287,6 +1318,12 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex; int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows; int32_t numOfRows = pInput->numOfRows;
if (IS_NULL_TYPE(type)) {
GET_RES_INFO(pCtx)->isNullRes = 1;
numOfElem = 1;
goto _stddev_over;
}
switch (type) { switch (type) {
case TSDB_DATA_TYPE_TINYINT: { case TSDB_DATA_TYPE_TINYINT: {
int8_t* plist = (int8_t*)pCol->pData; int8_t* plist = (int8_t*)pCol->pData;
@ -1384,6 +1421,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
break; break;
} }
_stddev_over:
// data in the check operation are all null, not output // data in the check operation are all null, not output
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1943,6 +1981,19 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t lastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0;
char* in = GET_ROWCELL_INTERBUF(pResInfo);
colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes);
return pResInfo->numOfRes;
}
bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SDiffInfo); pEnv->calcMemSize = sizeof(SDiffInfo);
return true; return true;

11
source/libs/index/src/indexComm.c
View File

@ -316,46 +316,57 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int64_t*)src, *dst, -1); indexInt2str(*(int64_t*)src, *dst, -1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint8_t*)src, *dst, 1); indexInt2str(*(uint8_t*)src, *dst, 1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int8_t*)src, *dst, 1); indexInt2str(*(int8_t*)src, *dst, 1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int16_t*)src, *dst, -1); indexInt2str(*(int16_t*)src, *dst, -1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_USMALLINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint16_t*)src, *dst, -1); indexInt2str(*(uint16_t*)src, *dst, -1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int32_t*)src, *dst, -1); indexInt2str(*(int32_t*)src, *dst, -1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint32_t*)src, *dst, 1); indexInt2str(*(uint32_t*)src, *dst, 1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
sprintf(*dst, "%" PRIu64, *(uint64_t*)src); sprintf(*dst, "%" PRIu64, *(uint64_t*)src);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint64_t*)src, *dst, 1); indexInt2str(*(uint64_t*)src, *dst, 1);
tlen = strlen(*dst);
case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_FLOAT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
sprintf(*dst, "%.9lf", *(float*)src); sprintf(*dst, "%.9lf", *(float*)src);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
sprintf(*dst, "%.9lf", *(double*)src); sprintf(*dst, "%.9lf", *(double*)src);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));

37
source/libs/index/test/jsonUT.cc
View File

@ -66,6 +66,17 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp
indexMultiTermDestroy(terms); indexMultiTermDestroy(terms);
} }
static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
int8_t operType = DEL_VALUE) {
SIndexTerm* term =
indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, (int64_t)tableId);
indexMultiTermDestroy(terms);
}
static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType, static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType,
SArray** result) { SArray** result) {
std::string colName(colNam); std::string colName(colNam);
@ -581,37 +592,45 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
TEST_F(JsonEnv, testWriteJsonTfileAndCache_DOUBLE) { TEST_F(JsonEnv, testWriteJsonTfileAndCache_DOUBLE) {
{ {
double val = 10.0; double val = 10.0;
std::string colName("test1");
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i); WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i);
} }
} }
{ {
double val = 2.0; double val = 2.0;
std::string colName("test1");
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000); WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000);
} }
} }
{ {
SArray* res = NULL; SArray* res = NULL;
std::string colName("test1"); std::string colName("test1");
double val = 1.9; double val = 1.9;
Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
EXPECT_EQ(2000, taosArrayGetSize(res)); EXPECT_EQ(2000, taosArrayGetSize(res));
} }
{ {
SArray* res = NULL; SArray* res = NULL;
std::string colName("test1");
double val = 2.1; double val = 2.1;
Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
EXPECT_EQ(1000, taosArrayGetSize(res)); EXPECT_EQ(1000, taosArrayGetSize(res));
} }
{ {
std::string colName("test1");
SArray* res = NULL; SArray* res = NULL;
double val = 2.1; double val = 2.1;
Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
EXPECT_EQ(1000, taosArrayGetSize(res));
}
{
SArray* res = NULL;
double val = 10.0;
Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_EQUAL, &res);
EXPECT_EQ(2000, taosArrayGetSize(res));
}
{
SArray* res = NULL;
double val = 10.0;
Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_THAN, &res);
EXPECT_EQ(1000, taosArrayGetSize(res)); EXPECT_EQ(1000, taosArrayGetSize(res));
} }
} }

1
source/libs/parser/inc/sql.y
View File

@ -621,6 +621,7 @@ column_reference(A) ::= table_name(B) NK_DOT column_name(C).
pseudo_column(A) ::= ROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= ROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= TBNAME(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= TBNAME(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= table_name(B) NK_DOT TBNAME(C). { A = createRawExprNodeExt(pCxt, &B, &C, createFunctionNode(pCxt, &C, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B)))); }
pseudo_column(A) ::= QSTARTTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= QSTARTTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= QENDTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= QENDTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= WSTARTTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= WSTARTTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }

9
source/libs/parser/src/parAstCreater.c
View File

@ -155,9 +155,9 @@ static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t
return TSDB_CODE_SUCCESS == pCxt->errCode; return TSDB_CODE_SUCCESS == pCxt->errCode;
} }
static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool query) { static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool demandDb) {
if (NULL == pDbName) { if (NULL == pDbName) {
if (query && NULL == pCxt->pQueryCxt->db) { if (demandDb && NULL == pCxt->pQueryCxt->db) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED); pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DB_NOT_SPECIFIED);
} }
} else { } else {
@ -457,6 +457,8 @@ SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const STok
} }
if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) {
strcpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias); strcpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias);
} else if (QUERY_NODE_SET_OPERATOR == nodeType(pSubquery)) {
strcpy(((SSetOperator*)pSubquery)->stmtName, tempTable->table.tableAlias);
} }
return (SNode*)tempTable; return (SNode*)tempTable;
} }
@ -637,6 +639,7 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode*
setOp->opType = type; setOp->opType = type;
setOp->pLeft = pLeft; setOp->pLeft = pLeft;
setOp->pRight = pRight; setOp->pRight = pRight;
sprintf(setOp->stmtName, "%p", setOp);
return (SNode*)setOp; return (SNode*)setOp;
} }
@ -1140,7 +1143,7 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName, SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName,
SToken* pTableName, SNodeList* pCols, SNode* pOptions) { SToken* pTableName, SNodeList* pCols, SNode* pOptions) {
if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName) || !checkDbName(pCxt, NULL, true)) {
return NULL; return NULL;
} }
SCreateIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_INDEX_STMT); SCreateIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_INDEX_STMT);

20
source/libs/parser/src/parTranslater.c
View File

@ -766,6 +766,20 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc)
pCxt->pCurrStmt->hasRepeatScanFuncs = true; pCxt->pCurrStmt->hasRepeatScanFuncs = true;
} }
} }
if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsScanPseudoColumnFunc(pFunc->funcId)) {
if (0 == LIST_LENGTH(pFunc->pParameterList)) {
if (QUERY_NODE_REAL_TABLE != nodeType(pCxt->pCurrStmt->pFromTable)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_TBNAME);
}
} else {
SValueNode* pVal = nodesListGetNode(pFunc->pParameterList, 0);
STableNode* pTable = NULL;
pCxt->errCode = findTable(pCxt, pVal->literal, &pTable);
if (TSDB_CODE_SUCCESS == pCxt->errCode && (NULL == pTable || QUERY_NODE_REAL_TABLE != nodeType(pTable))) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_TBNAME);
}
}
}
return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
} }
@ -1758,12 +1772,13 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
return code; return code;
} }
static SNode* createSetOperProject(SNode* pNode) { static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) {
SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) { if (NULL == pCol) {
return NULL; return NULL;
} }
pCol->node.resType = ((SExprNode*)pNode)->resType; pCol->node.resType = ((SExprNode*)pNode)->resType;
strcpy(pCol->tableAlias, pTableAlias);
strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName); strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName);
strcpy(pCol->node.aliasName, pCol->colName); strcpy(pCol->node.aliasName, pCol->colName);
return (SNode*)pCol; return (SNode*)pCol;
@ -1817,7 +1832,8 @@ static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* p
} }
strcpy(pRightExpr->aliasName, pLeftExpr->aliasName); strcpy(pRightExpr->aliasName, pLeftExpr->aliasName);
pRightExpr->aliasName[strlen(pLeftExpr->aliasName)] = '\0'; pRightExpr->aliasName[strlen(pLeftExpr->aliasName)] = '\0';
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, createSetOperProject(pLeft))) { if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList,
createSetOperProject(pSetOperator->stmtName, pLeft))) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
} }

2
source/libs/parser/src/parUtil.c
View File

@ -158,6 +158,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Primary timestamp column cannot be dropped"; return "Primary timestamp column cannot be dropped";
case TSDB_CODE_PAR_INVALID_MODIFY_COL: case TSDB_CODE_PAR_INVALID_MODIFY_COL:
return "Only binary/nchar column length could be modified"; return "Only binary/nchar column length could be modified";
case TSDB_CODE_PAR_INVALID_TBNAME:
return "Invalid tbname pseudo column";
case TSDB_CODE_OUT_OF_MEMORY: case TSDB_CODE_OUT_OF_MEMORY:
return "Out of memory"; return "Out of memory";
default: default:

1766
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

6
source/libs/parser/test/parSelectTest.cpp
View File

@ -70,6 +70,12 @@ TEST_F(ParserSelectTest, pseudoColumn) {
run("SELECT _WSTARTTS, _WENDTS, COUNT(*) FROM t1 INTERVAL(10s)"); run("SELECT _WSTARTTS, _WENDTS, COUNT(*) FROM t1 INTERVAL(10s)");
} }
TEST_F(ParserSelectTest, pseudoColumnSemanticCheck) {
useDb("root", "test");
run("SELECT TBNAME FROM (SELECT * FROM st1s1)", TSDB_CODE_PAR_INVALID_TBNAME, PARSER_STAGE_TRANSLATE);
}
TEST_F(ParserSelectTest, multiResFunc) { TEST_F(ParserSelectTest, multiResFunc) {
useDb("root", "test"); useDb("root", "test");

1
source/libs/planner/inc/planInt.h
View File

@ -32,6 +32,7 @@ extern "C" {
#define planWarn(param, ...) qWarn("PLAN: " param, __VA_ARGS__) #define planWarn(param, ...) qWarn("PLAN: " param, __VA_ARGS__)
#define planInfo(param, ...) qInfo("PLAN: " param, __VA_ARGS__) #define planInfo(param, ...) qInfo("PLAN: " param, __VA_ARGS__)
#define planDebug(param, ...) qDebug("PLAN: " param, __VA_ARGS__) #define planDebug(param, ...) qDebug("PLAN: " param, __VA_ARGS__)
#define planDebugL(param, ...) qDebugL("PLAN: " param, __VA_ARGS__)
#define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) #define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__)
int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...);

12
source/libs/planner/src/planLogicCreater.c
View File

@ -310,12 +310,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable, static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable,
SLogicNode** pLogicNode) { SLogicNode** pLogicNode) {
int32_t code = createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode); return createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode);
if (TSDB_CODE_SUCCESS == code) {
SNode* pNode;
FOREACH(pNode, (*pLogicNode)->pTargets) { strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias); }
}
return code;
} }
static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable, static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable,
@ -879,7 +874,8 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = createColumnByProjections(pCxt, NULL, pSetOperator->pProjectionList, &pProject->node.pTargets); code = createColumnByProjections(pCxt, pSetOperator->stmtName, pSetOperator->pProjectionList,
&pProject->node.pTargets);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -933,7 +929,7 @@ static int32_t createSetOpLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetO
code = createSetOpAggLogicNode(pCxt, pSetOperator, &pSetOp); code = createSetOpAggLogicNode(pCxt, pSetOperator, &pSetOp);
break; break;
default: default:
code = -1; code = TSDB_CODE_FAILED;
break; break;
} }

62
source/libs/planner/src/planOptimizer.c
View File

@ -598,39 +598,63 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
return false; return false;
} }
static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) { static bool cpdContainPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(pCond)) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pCond;
} if (LOGIC_COND_TYPE_AND != pLogicCond->condType) {
return TSDB_CODE_SUCCESS; return false;
}
static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) {
if (LOGIC_COND_TYPE_AND != pOnCond->condType) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
} }
bool hasPrimaryKeyEqualCond = false; bool hasPrimaryKeyEqualCond = false;
SNode* pCond = NULL; SNode* pCond = NULL;
FOREACH(pCond, pOnCond->pParameterList) { FOREACH(pCond, pLogicCond->pParameterList) {
if (cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { if (cpdContainPrimaryKeyEqualCond(pJoin, pCond)) {
hasPrimaryKeyEqualCond = true; hasPrimaryKeyEqualCond = true;
break;
} }
} }
if (!hasPrimaryKeyEqualCond) { return hasPrimaryKeyEqualCond;
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } else {
return cpdIsPrimaryKeyEqualCond(pJoin, pCond);
} }
return TSDB_CODE_SUCCESS;
} }
// static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) {
// if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) {
// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
// }
// return TSDB_CODE_SUCCESS;
// }
// static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) {
// if (LOGIC_COND_TYPE_AND != pOnCond->condType) {
// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
// }
// bool hasPrimaryKeyEqualCond = false;
// SNode* pCond = NULL;
// FOREACH(pCond, pOnCond->pParameterList) {
// if (cpdIsPrimaryKeyEqualCond(pJoin, pCond)) {
// hasPrimaryKeyEqualCond = true;
// }
// }
// if (!hasPrimaryKeyEqualCond) {
// return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
// }
// return TSDB_CODE_SUCCESS;
// }
static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
if (NULL == pJoin->pOnConditions) { if (NULL == pJoin->pOnConditions) {
return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN); return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN);
} }
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) { if (!cpdContainPrimaryKeyEqualCond(pJoin, pJoin->pOnConditions)) {
return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions); return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL);
} else {
return cpdCheckOpCond(pCxt, pJoin, pJoin->pOnConditions);
} }
return TSDB_CODE_SUCCESS;
// if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) {
// return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions);
// } else {
// return cpdCheckOpCond(pCxt, pJoin, pJoin->pOnConditions);
// }
} }
static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {

108
source/libs/planner/src/planSpliter.c
View File

@ -204,6 +204,75 @@ static int32_t ctjSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
return code; return code;
} }
static bool unionIsChildSubplan(SLogicNode* pLogicNode, int32_t groupId) {
if (QUERY_NODE_LOGIC_PLAN_EXCHANGE == nodeType(pLogicNode)) {
return ((SExchangeLogicNode*)pLogicNode)->srcGroupId == groupId;
}
SNode* pChild;
FOREACH(pChild, pLogicNode->pChildren) {
bool isChild = unionIsChildSubplan((SLogicNode*)pChild, groupId);
if (isChild) {
return isChild;
}
}
return false;
}
static int32_t unionMountSubplan(SLogicSubplan* pParent, SNodeList* pChildren) {
SNode* pChild = NULL;
WHERE_EACH(pChild, pChildren) {
if (unionIsChildSubplan(pParent->pNode, ((SLogicSubplan*)pChild)->id.groupId)) {
int32_t code = nodesListMakeAppend(&pParent->pChildren, pChild);
if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
ERASE_NODE(pChildren);
continue;
} else {
return code;
}
}
WHERE_NEXT;
}
return TSDB_CODE_SUCCESS;
}
static SLogicSubplan* unionCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) {
SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
if (NULL == pSubplan) {
return NULL;
}
pSubplan->id.groupId = pCxt->groupId;
pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
pSubplan->pNode = pNode;
return pSubplan;
}
static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubplan, SLogicNode* pSplitNode) {
SNodeList* pSubplanChildren = pUnionSubplan->pChildren;
pUnionSubplan->pChildren = NULL;
int32_t code = TSDB_CODE_SUCCESS;
SNode* pChild = NULL;
FOREACH(pChild, pSplitNode->pChildren) {
SLogicSubplan* pNewSubplan = unionCreateSubplan(pCxt, (SLogicNode*)pChild);
code = nodesListMakeStrictAppend(&pUnionSubplan->pChildren, pNewSubplan);
if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
code = unionMountSubplan(pNewSubplan, pSubplanChildren);
}
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {
nodesDestroyList(pSubplanChildren);
DESTORY_LIST(pSplitNode->pChildren);
}
return code;
}
static SLogicNode* uaMatchByNode(SLogicNode* pNode) { static SLogicNode* uaMatchByNode(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) {
return pNode; return pNode;
@ -227,17 +296,6 @@ static bool uaFindSplitNode(SLogicSubplan* pSubplan, SUaInfo* pInfo) {
return NULL != pSplitNode; return NULL != pSplitNode;
} }
static SLogicSubplan* uaCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) {
SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN);
if (NULL == pSubplan) {
return NULL;
}
pSubplan->id.groupId = pCxt->groupId;
pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
pSubplan->pNode = pNode;
return pSubplan;
}
static int32_t uaCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SProjectLogicNode* pProject) { static int32_t uaCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SProjectLogicNode* pProject) {
SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE);
if (NULL == pExchange) { if (NULL == pExchange) {
@ -276,20 +334,8 @@ static int32_t uaSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t code = TSDB_CODE_SUCCESS; int32_t code = unionSplitSubplan(pCxt, info.pSubplan, (SLogicNode*)info.pProject);
SNode* pChild = NULL;
FOREACH(pChild, info.pProject->node.pChildren) {
code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild));
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
} else {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {
nodesClearList(info.pProject->node.pChildren);
info.pProject->node.pChildren = NULL;
code = uaCreateExchangeNode(pCxt, info.pSubplan, info.pProject); code = uaCreateExchangeNode(pCxt, info.pSubplan, info.pProject);
} }
++(pCxt->groupId); ++(pCxt->groupId);
@ -343,20 +389,8 @@ static int32_t unSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t code = TSDB_CODE_SUCCESS; int32_t code = unionSplitSubplan(pCxt, info.pSubplan, (SLogicNode*)info.pAgg);
SNode* pChild = NULL;
FOREACH(pChild, info.pAgg->node.pChildren) {
code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild));
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(NULL);
} else {
break;
}
}
if (TSDB_CODE_SUCCESS == code) {
nodesClearList(info.pAgg->node.pChildren);
info.pAgg->node.pChildren = NULL;
code = unCreateExchangeNode(pCxt, info.pSubplan, info.pAgg); code = unCreateExchangeNode(pCxt, info.pSubplan, info.pAgg);
} }
++(pCxt->groupId); ++(pCxt->groupId);

8
source/libs/planner/test/planJoinTest.cpp
View File

@ -30,6 +30,14 @@ TEST_F(PlanJoinTest, basic) {
run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts"); run("SELECT t1.c1, t2.c1 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts");
} }
TEST_F(PlanJoinTest, complex) {
useDb("root", "test");
run("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 "
"WHERE t1.ts = t2.ts AND t1.c1 BETWEEN -10 AND 10 AND t2.c1 BETWEEN -100 AND 100 AND "
"(t1.c2 LIKE 'nchar%' OR t1.c1 = 0 OR t2.c2 LIKE 'nchar%' OR t2.c1 = 0)");
}
TEST_F(PlanJoinTest, withWhere) { TEST_F(PlanJoinTest, withWhere) {
useDb("root", "test"); useDb("root", "test");

26
source/libs/planner/test/planSetOpTest.cpp
View File

@ -23,11 +23,33 @@ class PlanSetOpTest : public PlannerTestBase {};
TEST_F(PlanSetOpTest, unionAll) { TEST_F(PlanSetOpTest, unionAll) {
useDb("root", "test"); useDb("root", "test");
run("select c1, c2 from t1 where c1 > 10 union all select c1, c2 from t1 where c1 > 20"); run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20");
}
TEST_F(PlanSetOpTest, unionAllSubquery) {
useDb("root", "test");
run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION ALL SELECT c1, c2 FROM t1)");
} }
TEST_F(PlanSetOpTest, union) { TEST_F(PlanSetOpTest, union) {
useDb("root", "test"); useDb("root", "test");
run("select c1, c2 from t1 where c1 > 10 union select c1, c2 from t1 where c1 > 20"); run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION SELECT c1, c2 FROM t1 WHERE c1 > 20");
}
TEST_F(PlanSetOpTest, unionContainJoin) {
useDb("root", "test");
run("SELECT t1.c1 FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts "
"WHERE t1.c1 IS NOT NULL GROUP BY t1.c1 HAVING t1.c1 IS NOT NULL "
"UNION "
"SELECT t1.c1 FROM st1s1 t1 join st1s2 t2 on t1.ts = t2.ts "
"WHERE t1.c1 IS NOT NULL GROUP BY t1.c1 HAVING t1.c1 IS NOT NULL");
}
TEST_F(PlanSetOpTest, unionSubquery) {
useDb("root", "test");
run("SELECT * FROM (SELECT c1, c2 FROM t1 UNION SELECT c1, c2 FROM t1)");
} }

62
source/libs/qworker/src/qworker.c
View File

@ -110,9 +110,9 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) {
QW_LOCK(QW_READ, &mgmt->schLock); QW_LOCK(QW_READ, &mgmt->schLock);
QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash)); /*QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash));*/
void * key = NULL; void *key = NULL;
size_t keyLen = 0; size_t keyLen = 0;
int32_t i = 0; int32_t i = 0;
SQWSchStatus *sch = NULL; SQWSchStatus *sch = NULL;
@ -127,7 +127,7 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) {
QW_UNLOCK(QW_READ, &mgmt->schLock); QW_UNLOCK(QW_READ, &mgmt->schLock);
QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash)); /*QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash));*/
} }
char *qwPhaseStr(int32_t phase) { char *qwPhaseStr(int32_t phase) {
@ -462,7 +462,7 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) {
} }
int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { int32_t qwDropTaskStatus(QW_FPARAMS_DEF) {
SQWSchStatus * sch = NULL; SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL; SQWTaskStatus *task = NULL;
int32_t code = 0; int32_t code = 0;
@ -499,7 +499,7 @@ _return:
} }
int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status) { int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status) {
SQWSchStatus * sch = NULL; SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL; SQWTaskStatus *task = NULL;
int32_t code = 0; int32_t code = 0;
@ -550,11 +550,11 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) {
int32_t code = 0; int32_t code = 0;
bool qcontinue = true; bool qcontinue = true;
SSDataBlock * pRes = NULL; SSDataBlock *pRes = NULL;
uint64_t useconds = 0; uint64_t useconds = 0;
int32_t i = 0; int32_t i = 0;
int32_t execNum = 0; int32_t execNum = 0;
qTaskInfo_t * taskHandle = &ctx->taskHandle; qTaskInfo_t *taskHandle = &ctx->taskHandle;
DataSinkHandle sinkHandle = ctx->sinkHandle; DataSinkHandle sinkHandle = ctx->sinkHandle;
while (true) { while (true) {
@ -632,7 +632,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo)
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }
void * key = NULL; void *key = NULL;
size_t keyLen = 0; size_t keyLen = 0;
int32_t i = 0; int32_t i = 0;
STaskStatus status = {0}; STaskStatus status = {0};
@ -720,7 +720,7 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void
int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) {
int32_t code = 0; int32_t code = 0;
SQWTaskCtx * ctx = NULL; SQWTaskCtx *ctx = NULL;
SRpcHandleInfo *dropConnection = NULL; SRpcHandleInfo *dropConnection = NULL;
SRpcHandleInfo *cancelConnection = NULL; SRpcHandleInfo *cancelConnection = NULL;
@ -927,11 +927,11 @@ _return:
int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) {
int32_t code = 0; int32_t code = 0;
bool queryRsped = false; bool queryRsped = false;
SSubplan* plan = NULL; SSubplan *plan = NULL;
SQWPhaseInput input = {0}; SQWPhaseInput input = {0};
qTaskInfo_t pTaskInfo = NULL; qTaskInfo_t pTaskInfo = NULL;
DataSinkHandle sinkHandle = NULL; DataSinkHandle sinkHandle = NULL;
SQWTaskCtx * ctx = NULL; SQWTaskCtx *ctx = NULL;
QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo));
@ -944,7 +944,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex
ctx->ctrlConnInfo = qwMsg->connInfo; ctx->ctrlConnInfo = qwMsg->connInfo;
QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); /*QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg);*/
code = qStringToSubplan(qwMsg->msg, &plan); code = qStringToSubplan(qwMsg->msg, &plan);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
@ -1055,10 +1055,10 @@ _return:
} }
int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
SQWTaskCtx * ctx = NULL; SQWTaskCtx *ctx = NULL;
int32_t code = 0; int32_t code = 0;
SQWPhaseInput input = {0}; SQWPhaseInput input = {0};
void * rsp = NULL; void *rsp = NULL;
int32_t dataLen = 0; int32_t dataLen = 0;
bool queryEnd = false; bool queryEnd = false;
@ -1138,8 +1138,8 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
int32_t code = 0; int32_t code = 0;
int32_t dataLen = 0; int32_t dataLen = 0;
bool locked = false; bool locked = false;
SQWTaskCtx * ctx = NULL; SQWTaskCtx *ctx = NULL;
void * rsp = NULL; void *rsp = NULL;
SQWPhaseInput input = {0}; SQWPhaseInput input = {0};
QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL)); QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL));
@ -1274,7 +1274,7 @@ _return:
int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
int32_t code = 0; int32_t code = 0;
SSchedulerHbRsp rsp = {0}; SSchedulerHbRsp rsp = {0};
SQWSchStatus * sch = NULL; SQWSchStatus *sch = NULL;
QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch));
@ -1300,7 +1300,7 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re
int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
int32_t code = 0; int32_t code = 0;
SSchedulerHbRsp rsp = {0}; SSchedulerHbRsp rsp = {0};
SQWSchStatus * sch = NULL; SQWSchStatus *sch = NULL;
if (qwMsg->code) { if (qwMsg->code) {
QW_RET(qwProcessHbLinkBroken(mgmt, qwMsg, req)); QW_RET(qwProcessHbLinkBroken(mgmt, qwMsg, req));
@ -1338,13 +1338,13 @@ _return:
qwMsg->connInfo.handle = NULL; qwMsg->connInfo.handle = NULL;
} }
QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));*/
QW_RET(TSDB_CODE_SUCCESS); QW_RET(TSDB_CODE_SUCCESS);
} }
void qwProcessHbTimerEvent(void *param, void *tmrId) { void qwProcessHbTimerEvent(void *param, void *tmrId) {
SQWHbParam* hbParam = (SQWHbParam*)param; SQWHbParam *hbParam = (SQWHbParam *)param;
if (hbParam->qwrId != atomic_load_32(&gQwMgmt.qwRef)) { if (hbParam->qwrId != atomic_load_32(&gQwMgmt.qwRef)) {
return; return;
} }
@ -1359,7 +1359,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
SQWSchStatus *sch = NULL; SQWSchStatus *sch = NULL;
int32_t taskNum = 0; int32_t taskNum = 0;
SQWHbInfo * rspList = NULL; SQWHbInfo *rspList = NULL;
int32_t code = 0; int32_t code = 0;
qwDbgDumpMgmtInfo(mgmt); qwDbgDumpMgmtInfo(mgmt);
@ -1383,7 +1383,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
return; return;
} }
void * key = NULL; void *key = NULL;
size_t keyLen = 0; size_t keyLen = 0;
int32_t i = 0; int32_t i = 0;
@ -1413,8 +1413,8 @@ _return:
for (int32_t j = 0; j < i; ++j) { for (int32_t j = 0; j < i; ++j) {
qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code);
QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code), /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),*/
(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0)); /*(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));*/
tFreeSSchedulerHbRsp(&rspList[j].rsp); tFreeSSchedulerHbRsp(&rspList[j].rsp);
} }
@ -1428,14 +1428,12 @@ void qwCloseRef(void) {
taosWLockLatch(&gQwMgmt.lock); taosWLockLatch(&gQwMgmt.lock);
if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) { if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) {
taosCloseRef(gQwMgmt.qwRef); taosCloseRef(gQwMgmt.qwRef);
gQwMgmt.qwRef= -1; gQwMgmt.qwRef = -1;
} }
taosWUnLockLatch(&gQwMgmt.lock); taosWUnLockLatch(&gQwMgmt.lock);
} }
void qwDestroySchStatus(SQWSchStatus *pStatus) { void qwDestroySchStatus(SQWSchStatus *pStatus) { taosHashCleanup(pStatus->tasksHash); }
taosHashCleanup(pStatus->tasksHash);
}
void qwDestroyImpl(void *pMgmt) { void qwDestroyImpl(void *pMgmt) {
SQWorker *mgmt = (SQWorker *)pMgmt; SQWorker *mgmt = (SQWorker *)pMgmt;
@ -1467,7 +1465,7 @@ void qwDestroyImpl(void *pMgmt) {
int32_t qwOpenRef(void) { int32_t qwOpenRef(void) {
taosWLockLatch(&gQwMgmt.lock); taosWLockLatch(&gQwMgmt.lock);
if (gQwMgmt.qwRef < 0) { if (gQwMgmt.qwRef < 0) {
gQwMgmt.qwRef= taosOpenRef(100, qwDestroyImpl); gQwMgmt.qwRef = taosOpenRef(100, qwDestroyImpl);
if (gQwMgmt.qwRef < 0) { if (gQwMgmt.qwRef < 0) {
taosWUnLockLatch(&gQwMgmt.lock); taosWUnLockLatch(&gQwMgmt.lock);
qError("init qworker ref failed"); qError("init qworker ref failed");
@ -1577,7 +1575,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW
SQWHbParam *param = NULL; SQWHbParam *param = NULL;
qwSetHbParam(mgmt->refId, &param); qwSetHbParam(mgmt->refId, &param);
mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void*)param, mgmt->timer); mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void *)param, mgmt->timer);
if (NULL == mgmt->hbTimer) { if (NULL == mgmt->hbTimer) {
qError("start hb timer failed"); qError("start hb timer failed");
QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -1678,7 +1676,7 @@ int32_t qwUpdateSchLastAccess(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64
} }
int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) { int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) {
SQWSchStatus * sch = NULL; SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL; SQWTaskStatus *task = NULL;
int32_t code = 0; int32_t code = 0;
@ -1705,7 +1703,7 @@ int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId
} }
int32_t qwCancelTask(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { int32_t qwCancelTask(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) {
SQWSchStatus * sch = NULL; SQWSchStatus *sch = NULL;
SQWTaskStatus *task = NULL; SQWTaskStatus *task = NULL;
int32_t code = 0; int32_t code = 0;

2
source/libs/tdb/CMakeLists.txt
View File

@ -7,7 +7,7 @@ target_sources(tdb
"src/db/tdbUtil.c" "src/db/tdbUtil.c"
"src/db/tdbBtree.c" "src/db/tdbBtree.c"
"src/db/tdbDb.c" "src/db/tdbDb.c"
"src/db/tdbEnv.c" "src/db/tdbTable.c"
"src/db/tdbTxn.c" "src/db/tdbTxn.c"
"src/db/tdbPage.c" "src/db/tdbPage.c"
"src/db/tdbOs.c" "src/db/tdbOs.c"

66
source/libs/tdb/inc/tdb.h
View File

@ -22,51 +22,51 @@
extern "C" { extern "C" {
#endif #endif
typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int32_t kLen1, const void *pKey2, int32_t kLen2);
// exposed types // exposed types
typedef struct STEnv TENV;
typedef struct STDB TDB; typedef struct STDB TDB;
typedef struct STDBC TDBC; typedef struct STTB TTB;
typedef struct STBC TBC;
typedef struct STxn TXN; typedef struct STxn TXN;
// TENV
int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv);
int tdbEnvClose(TENV *pEnv);
int tdbBegin(TENV *pEnv, TXN *pTxn);
int tdbCommit(TENV *pEnv, TXN *pTxn);
// TDB // TDB
int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb);
int tdbClose(TDB *pDb); int32_t tdbClose(TDB *pDb);
int tdbDrop(TDB *pDb); int32_t tdbBegin(TDB *pDb, TXN *pTxn);
int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); int32_t tdbCommit(TDB *pDb, TXN *pTxn);
int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn);
int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen);
int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
// TDBC // TTB
int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); int32_t tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb);
int tdbDbcClose(TDBC *pDbc); int32_t tdbTbClose(TTB *pTb);
int tdbDbcIsValid(TDBC *pDbc); int32_t tdbTbDrop(TTB *pTb);
int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); int32_t tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn);
int tdbDbcMoveToFirst(TDBC *pDbc); int32_t tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn);
int tdbDbcMoveToLast(TDBC *pDbc); int32_t tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
int tdbDbcMoveToNext(TDBC *pDbc); int32_t tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen);
int tdbDbcMoveToPrev(TDBC *pDbc); int32_t tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen);
int tdbDbcDelete(TDBC *pDbc); // TBC
int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); int32_t tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn);
int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert); int32_t tdbTbcClose(TBC *pTbc);
int32_t tdbTbcIsValid(TBC *pTbc);
int32_t tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c);
int32_t tdbTbcMoveToFirst(TBC *pTbc);
int32_t tdbTbcMoveToLast(TBC *pTbc);
int32_t tdbTbcMoveToNext(TBC *pTbc);
int32_t tdbTbcMoveToPrev(TBC *pTbc);
int32_t tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen);
int32_t tdbTbcDelete(TBC *pTbc);
int32_t tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen);
int32_t tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert);
// TXN // TXN
#define TDB_TXN_WRITE 0x1 #define TDB_TXN_WRITE 0x1
#define TDB_TXN_READ_UNCOMMITTED 0x2 #define TDB_TXN_READ_UNCOMMITTED 0x2
int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, int32_t tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *),
int flags); void *xArg, int flags);
int tdbTxnClose(TXN *pTxn); int32_t tdbTxnClose(TXN *pTxn);
// other // other
void tdbFree(void *); void tdbFree(void *);

220
source/libs/tdb/src/db/tdbDb.c
View File

@ -15,134 +15,164 @@
#include "tdbInt.h" #include "tdbInt.h"
struct STDB { int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb) {
TENV *pEnv;
SBTree *pBt;
};
struct STDBC {
SBTC btc;
};
int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb) {
TDB *pDb; TDB *pDb;
SPager *pPager; int dsize;
int zsize;
int tsize;
u8 *pPtr;
int ret; int ret;
char fFullName[TDB_FILENAME_LEN];
SPage *pPage;
SPgno pgno;
*ppDb = NULL; *ppDb = NULL;
pDb = (TDB *)tdbOsCalloc(1, sizeof(*pDb)); dsize = strlen(dbname);
if (pDb == NULL) { zsize = sizeof(*pDb) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3;
pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) {
return -1; return -1;
} }
// pDb->pEnv pDb = (TDB *)pPtr;
pDb->pEnv = pEnv; pPtr += sizeof(*pDb);
// pDb->rootDir
pDb->dbName = pPtr;
memcpy(pDb->dbName, dbname, dsize);
pDb->dbName[dsize] = '\0';
pPtr = pPtr + dsize + 1;
// pDb->jfname
pDb->jnName = pPtr;
memcpy(pDb->jnName, dbname, dsize);
pDb->jnName[dsize] = '/';
memcpy(pDb->jnName + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME));
pDb->jnName[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0';
pPager = tdbEnvGetPager(pEnv, fname); pDb->jfd = -1;
if (pPager == NULL) {
snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));
ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
tdbEnvAddPager(pEnv, pPager); pDb->nPgrHash = 8;
} tsize = sizeof(SPager *) * pDb->nPgrHash;
pDb->pgrHash = tdbOsMalloc(tsize);
ASSERT(pPager != NULL); if (pDb->pgrHash == NULL) {
// pDb->pBt
ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pDb->pBt));
if (ret < 0) {
return -1; return -1;
} }
memset(pDb->pgrHash, 0, tsize);
mkdir(dbname, 0755);
*ppDb = pDb; *ppDb = pDb;
return 0; return 0;
} }
int tdbClose(TDB *pDb) { int tdbClose(TDB *pDb) {
SPager *pPager;
if (pDb) { if (pDb) {
tdbBtreeClose(pDb->pBt); for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) {
pDb->pgrList = pPager->pNext;
tdbPagerClose(pPager);
}
tdbPCacheClose(pDb->pCache);
tdbOsFree(pDb->pgrHash);
tdbOsFree(pDb); tdbOsFree(pDb);
} }
return 0; return 0;
} }
int tdbDrop(TDB *pDb) { int tdbBegin(TDB *pDb, TXN *pTxn) {
// TODO SPager *pPager;
return 0;
}
int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn);
}
int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); }
int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn);
}
int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {
return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen);
}
int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) {
return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen);
}
int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn) {
int ret; int ret;
TDBC *pDbc = NULL;
*ppDbc = NULL; for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
pDbc = (TDBC *)tdbOsMalloc(sizeof(*pDbc)); ret = tdbPagerBegin(pPager, pTxn);
if (pDbc == NULL) { if (ret < 0) {
ASSERT(0);
return -1; return -1;
} }
tdbBtcOpen(&pDbc->btc, pDb->pBt, pTxn);
*ppDbc = pDbc;
return 0;
}
int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pDbc->btc, pKey, kLen, c); }
int tdbDbcMoveToFirst(TDBC *pDbc) { return tdbBtcMoveToFirst(&pDbc->btc); }
int tdbDbcMoveToLast(TDBC *pDbc) { return tdbBtcMoveToLast(&pDbc->btc); }
int tdbDbcMoveToNext(TDBC *pDbc) { return tdbBtcMoveToNext(&pDbc->btc); }
int tdbDbcMoveToPrev(TDBC *pDbc) { return tdbBtcMoveToPrev(&pDbc->btc); }
int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) {
return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen);
}
int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); }
int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen);
}
int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) {
return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert);
}
int tdbDbcClose(TDBC *pDbc) {
if (pDbc) {
tdbBtcClose(&pDbc->btc);
tdbOsFree(pDbc);
} }
return 0; return 0;
} }
int tdbDbcIsValid(TDBC *pDbc) { return tdbBtcIsValid(&pDbc->btc); } int tdbCommit(TDB *pDb, TXN *pTxn) {
SPager *pPager;
int ret;
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerCommit(pPager, pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
}
}
return 0;
}
SPager *tdbEnvGetPager(TDB *pDb, const char *fname) {
u32 hash;
SPager **ppPager;
hash = tdbCstringHash(fname);
ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];
for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) {
}
return *ppPager;
}
void tdbEnvAddPager(TDB *pDb, SPager *pPager) {
u32 hash;
SPager **ppPager;
// rehash if neccessary
if (pDb->nPager + 1 > pDb->nPgrHash) {
// TODO
}
// add to list
pPager->pNext = pDb->pgrList;
pDb->pgrList = pPager;
// add to hash
hash = tdbCstringHash(pPager->dbFileName);
ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];
pPager->pHashNext = *ppPager;
*ppPager = pPager;
// increase the counter
pDb->nPager++;
}
void tdbEnvRemovePager(TDB *pDb, SPager *pPager) {
u32 hash;
SPager **ppPager;
// remove from the list
for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) {
}
ASSERT(*ppPager == pPager);
*ppPager = pPager->pNext;
// remove from hash
hash = tdbCstringHash(pPager->dbFileName);
ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];
for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) {
}
ASSERT(*ppPager == pPager);
*ppPager = pPager->pNext;
// decrease the counter
pDb->nPager--;
// rehash if necessary
if (pDb->nPgrHash > 8 && pDb->nPager < pDb->nPgrHash / 2) {
// TODO
}
}

178
source/libs/tdb/src/db/tdbEnv.c
View File

@ -1,178 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv) {
TENV *pEnv;
int dsize;
int zsize;
int tsize;
u8 *pPtr;
int ret;
*ppEnv = NULL;
dsize = strlen(rootDir);
zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3;
pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) {
return -1;
}
pEnv = (TENV *)pPtr;
pPtr += sizeof(*pEnv);
// pEnv->rootDir
pEnv->rootDir = pPtr;
memcpy(pEnv->rootDir, rootDir, dsize);
pEnv->rootDir[dsize] = '\0';
pPtr = pPtr + dsize + 1;
// pEnv->jfname
pEnv->jfname = pPtr;
memcpy(pEnv->jfname, rootDir, dsize);
pEnv->jfname[dsize] = '/';
memcpy(pEnv->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME));
pEnv->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0';
pEnv->jfd = -1;
ret = tdbPCacheOpen(szPage, pages, &(pEnv->pCache));
if (ret < 0) {
return -1;
}
pEnv->nPgrHash = 8;
tsize = sizeof(SPager *) * pEnv->nPgrHash;
pEnv->pgrHash = tdbOsMalloc(tsize);
if (pEnv->pgrHash == NULL) {
return -1;
}
memset(pEnv->pgrHash, 0, tsize);
mkdir(rootDir, 0755);
*ppEnv = pEnv;
return 0;
}
int tdbEnvClose(TENV *pEnv) {
SPager *pPager;
if (pEnv) {
for (pPager = pEnv->pgrList; pPager; pPager = pEnv->pgrList) {
pEnv->pgrList = pPager->pNext;
tdbPagerClose(pPager);
}
tdbPCacheClose(pEnv->pCache);
tdbOsFree(pEnv->pgrHash);
tdbOsFree(pEnv);
}
return 0;
}
int tdbBegin(TENV *pEnv, TXN *pTxn) {
SPager *pPager;
int ret;
for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerBegin(pPager, pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
}
}
return 0;
}
int tdbCommit(TENV *pEnv, TXN *pTxn) {
SPager *pPager;
int ret;
for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerCommit(pPager, pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
}
}
return 0;
}
SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) {
u32 hash;
SPager **ppPager;
hash = tdbCstringHash(fname);
ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) {
}
return *ppPager;
}
void tdbEnvAddPager(TENV *pEnv, SPager *pPager) {
u32 hash;
SPager **ppPager;
// rehash if neccessary
if (pEnv->nPager + 1 > pEnv->nPgrHash) {
// TODO
}
// add to list
pPager->pNext = pEnv->pgrList;
pEnv->pgrList = pPager;
// add to hash
hash = tdbCstringHash(pPager->dbFileName);
ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
pPager->pHashNext = *ppPager;
*ppPager = pPager;
// increase the counter
pEnv->nPager++;
}
void tdbEnvRemovePager(TENV *pEnv, SPager *pPager) {
u32 hash;
SPager **ppPager;
// remove from the list
for (ppPager = &pEnv->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) {
}
ASSERT(*ppPager == pPager);
*ppPager = pPager->pNext;
// remove from hash
hash = tdbCstringHash(pPager->dbFileName);
ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) {
}
ASSERT(*ppPager == pPager);
*ppPager = pPager->pNext;
// decrease the counter
pEnv->nPager--;
// rehash if necessary
if (pEnv->nPgrHash > 8 && pEnv->nPager < pEnv->nPgrHash / 2) {
// TODO
}
}

148
source/libs/tdb/src/db/tdbTable.c Normal file
View File

@ -0,0 +1,148 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct STTB {
TDB *pEnv;
SBTree *pBt;
};
struct STBC {
SBTC btc;
};
int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) {
TTB *pTb;
SPager *pPager;
int ret;
char fFullName[TDB_FILENAME_LEN];
SPage *pPage;
SPgno pgno;
*ppTb = NULL;
pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb));
if (pTb == NULL) {
return -1;
}
// pTb->pEnv
pTb->pEnv = pEnv;
pPager = tdbEnvGetPager(pEnv, tbname);
if (pPager == NULL) {
snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname);
ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
if (ret < 0) {
return -1;
}
tdbEnvAddPager(pEnv, pPager);
}
ASSERT(pPager != NULL);
// pTb->pBt
ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pTb->pBt));
if (ret < 0) {
return -1;
}
*ppTb = pTb;
return 0;
}
int tdbTbClose(TTB *pTb) {
if (pTb) {
tdbBtreeClose(pTb->pBt);
tdbOsFree(pTb);
}
return 0;
}
int tdbTbDrop(TTB *pTb) {
// TODO
return 0;
}
int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
return tdbBtreeInsert(pTb->pBt, pKey, keyLen, pVal, valLen, pTxn);
}
int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pTb->pBt, pKey, kLen, pTxn); }
int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
return tdbBtreeUpsert(pTb->pBt, pKey, kLen, pVal, vLen, pTxn);
}
int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen) {
return tdbBtreeGet(pTb->pBt, pKey, kLen, ppVal, vLen);
}
int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) {
return tdbBtreePGet(pTb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen);
}
int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn) {
int ret;
TBC *pTbc = NULL;
*ppTbc = NULL;
pTbc = (TBC *)tdbOsMalloc(sizeof(*pTbc));
if (pTbc == NULL) {
return -1;
}
tdbBtcOpen(&pTbc->btc, pTb->pBt, pTxn);
*ppTbc = pTbc;
return 0;
}
int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pTbc->btc, pKey, kLen, c); }
int tdbTbcMoveToFirst(TBC *pTbc) { return tdbBtcMoveToFirst(&pTbc->btc); }
int tdbTbcMoveToLast(TBC *pTbc) { return tdbBtcMoveToLast(&pTbc->btc); }
int tdbTbcMoveToNext(TBC *pTbc) { return tdbBtcMoveToNext(&pTbc->btc); }
int tdbTbcMoveToPrev(TBC *pTbc) { return tdbBtcMoveToPrev(&pTbc->btc); }
int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) {
return tdbBtcGet(&pTbc->btc, ppKey, pkLen, ppVal, pvLen);
}
int tdbTbcDelete(TBC *pTbc) { return tdbBtcDelete(&pTbc->btc); }
int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
return tdbBtreeNext(&pTbc->btc, ppKey, kLen, ppVal, vLen);
}
int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert) {
return tdbBtcUpsert(&pTbc->btc, pKey, nKey, pData, nData, insert);
}
int tdbTbcClose(TBC *pTbc) {
if (pTbc) {
tdbBtcClose(&pTbc->btc);
tdbOsFree(pTbc);
}
return 0;
}
int tdbTbcIsValid(TBC *pTbc) { return tdbBtcIsValid(&pTbc->btc); }

16
source/libs/tdb/src/inc/tdbInt.h
View File

@ -103,9 +103,9 @@ typedef struct SPage SPage;
#define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED) #define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED)
// tdbEnv.c ==================================== // tdbEnv.c ====================================
void tdbEnvAddPager(TENV *pEnv, SPager *pPager); void tdbEnvAddPager(TDB *pEnv, SPager *pPager);
void tdbEnvRemovePager(TENV *pEnv, SPager *pPager); void tdbEnvRemovePager(TDB *pEnv, SPager *pPager);
SPager *tdbEnvGetPager(TENV *pEnv, const char *fname); SPager *tdbEnvGetPager(TDB *pEnv, const char *fname);
// tdbBtree.c ==================================== // tdbBtree.c ====================================
typedef struct SBTree SBTree; typedef struct SBTree SBTree;
@ -334,9 +334,9 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {
return pCell; return pCell;
} }
struct STEnv { struct STDB {
char *rootDir; char *dbName;
char *jfname; char *jnName;
int jfd; int jfd;
SPCache *pCache; SPCache *pCache;
SPager *pgrList; SPager *pgrList;
@ -357,8 +357,8 @@ struct SPager {
SPgno dbOrigSize; SPgno dbOrigSize;
SPage *pDirty; SPage *pDirty;
u8 inTran; u8 inTran;
SPager *pNext; // used by TENV SPager *pNext; // used by TDB
SPager *pHashNext; // used by TENV SPager *pHashNext; // used by TDB
}; };
#ifdef __cplusplus #ifdef __cplusplus

104
source/libs/tdb/test/tdbTest.cpp
View File

@ -117,8 +117,8 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in
TEST(tdb_test, simple_insert1) { TEST(tdb_test, simple_insert1) {
int ret; int ret;
TENV *pEnv; TDB *pEnv;
TDB *pDb; TTB *pDb;
tdb_cmpr_fn_t compFunc; tdb_cmpr_fn_t compFunc;
int nData = 1000000; int nData = 1000000;
TXN txn; TXN txn;
@ -126,12 +126,12 @@ TEST(tdb_test, simple_insert1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); ret = tdbOpen("tdb", 4096, 64, &pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
compFunc = tKeyCmpr; compFunc = tKeyCmpr;
ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
{ {
@ -152,7 +152,7 @@ TEST(tdb_test, simple_insert1) {
for (int iData = 1; iData <= nData; iData++) { for (int iData = 1; iData <= nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(val, "value%d", iData); sprintf(val, "value%d", iData);
ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// if pool is full, commit the transaction and start a new one // if pool is full, commit the transaction and start a new one
@ -181,7 +181,7 @@ TEST(tdb_test, simple_insert1) {
sprintf(key, "key%d", i); sprintf(key, "key%d", i);
sprintf(val, "value%d", i); sprintf(val, "value%d", i);
ret = tdbGet(pDb, key, strlen(key), &pVal, &vLen); ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen);
ASSERT(ret == 0); ASSERT(ret == 0);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
@ -193,19 +193,19 @@ TEST(tdb_test, simple_insert1) {
} }
{ // Iterate to query the DB data { // Iterate to query the DB data
TDBC *pDBC; TBC *pDBC;
void *pKey = NULL; void *pKey = NULL;
void *pVal = NULL; void *pVal = NULL;
int vLen, kLen; int vLen, kLen;
int count = 0; int count = 0;
ret = tdbDbcOpen(pDb, &pDBC, NULL); ret = tdbTbcOpen(pDb, &pDBC, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
tdbDbcMoveToFirst(pDBC); tdbTbcMoveToFirst(pDBC);
for (;;) { for (;;) {
ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break; if (ret < 0) break;
// std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " ";
@ -217,28 +217,28 @@ TEST(tdb_test, simple_insert1) {
GTEST_ASSERT_EQ(count, nData); GTEST_ASSERT_EQ(count, nData);
tdbDbcClose(pDBC); tdbTbcClose(pDBC);
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
} }
} }
ret = tdbDrop(pDb); ret = tdbTbDrop(pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Close a database // Close a database
tdbClose(pDb); tdbTbClose(pDb);
// Close Env // Close Env
ret = tdbEnvClose(pEnv); ret = tdbClose(pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
TEST(tdb_test, simple_insert2) { TEST(tdb_test, simple_insert2) {
int ret; int ret;
TENV *pEnv; TDB *pEnv;
TDB *pDb; TTB *pDb;
tdb_cmpr_fn_t compFunc; tdb_cmpr_fn_t compFunc;
int nData = 1000000; int nData = 1000000;
TXN txn; TXN txn;
@ -246,12 +246,12 @@ TEST(tdb_test, simple_insert2) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbEnvOpen("tdb", 1024, 10, &pEnv); ret = tdbOpen("tdb", 1024, 10, &pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
compFunc = tDefaultKeyCmpr; compFunc = tDefaultKeyCmpr;
ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
{ {
@ -271,24 +271,24 @@ TEST(tdb_test, simple_insert2) {
for (int iData = 1; iData <= nData; iData++) { for (int iData = 1; iData <= nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(val, "value%d", iData); sprintf(val, "value%d", iData);
ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
{ // Iterate to query the DB data { // Iterate to query the DB data
TDBC *pDBC; TBC *pDBC;
void *pKey = NULL; void *pKey = NULL;
void *pVal = NULL; void *pVal = NULL;
int vLen, kLen; int vLen, kLen;
int count = 0; int count = 0;
ret = tdbDbcOpen(pDb, &pDBC, NULL); ret = tdbTbcOpen(pDb, &pDBC, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
tdbDbcMoveToFirst(pDBC); tdbTbcMoveToFirst(pDBC);
for (;;) { for (;;) {
ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break; if (ret < 0) break;
// std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " ";
@ -300,7 +300,7 @@ TEST(tdb_test, simple_insert2) {
GTEST_ASSERT_EQ(count, nData); GTEST_ASSERT_EQ(count, nData);
tdbDbcClose(pDBC); tdbTbcClose(pDBC);
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
@ -311,29 +311,29 @@ TEST(tdb_test, simple_insert2) {
tdbCommit(pEnv, &txn); tdbCommit(pEnv, &txn);
tdbTxnClose(&txn); tdbTxnClose(&txn);
ret = tdbDrop(pDb); ret = tdbTbDrop(pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Close a database // Close a database
tdbClose(pDb); tdbTbClose(pDb);
// Close Env // Close Env
ret = tdbEnvClose(pEnv); ret = tdbClose(pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
TEST(tdb_test, simple_delete1) { TEST(tdb_test, simple_delete1) {
int ret; int ret;
TDB *pDb; TTB *pDb;
char key[128]; char key[128];
char data[128]; char data[128];
TXN txn; TXN txn;
TENV *pEnv; TDB *pEnv;
SPoolMem *pPool; SPoolMem *pPool;
void *pKey = NULL; void *pKey = NULL;
void *pData = NULL; void *pData = NULL;
int nKey; int nKey;
TDBC *pDbc; TBC *pDbc;
int nData; int nData;
int nKV = 69; int nKV = 69;
@ -342,11 +342,11 @@ TEST(tdb_test, simple_delete1) {
pPool = openPool(); pPool = openPool();
// open env // open env
ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); ret = tdbOpen("tdb", 1024, 256, &pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// open database // open database
ret = tdbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); ret = tdbTbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
@ -356,7 +356,7 @@ TEST(tdb_test, simple_delete1) {
for (int iData = 0; iData < nKV; iData++) { for (int iData = 0; iData < nKV; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d", iData); sprintf(data, "data%d", iData);
ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
@ -365,7 +365,7 @@ TEST(tdb_test, simple_delete1) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d", iData); sprintf(data, "data%d", iData);
ret = tdbGet(pDb, key, strlen(key), &pData, &nData); ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0);
} }
@ -374,7 +374,7 @@ TEST(tdb_test, simple_delete1) {
for (int iData = nKV - 1; iData > 30; iData--) { for (int iData = nKV - 1; iData > 30; iData--) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
ret = tdbDelete(pDb, key, strlen(key), &txn); ret = tdbTbDelete(pDb, key, strlen(key), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
@ -382,7 +382,7 @@ TEST(tdb_test, simple_delete1) {
for (int iData = 0; iData < nKV; iData++) { for (int iData = 0; iData < nKV; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
ret = tdbGet(pDb, key, strlen(key), &pData, &nData); ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData);
if (iData <= 30) { if (iData <= 30) {
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} else { } else {
@ -391,15 +391,15 @@ TEST(tdb_test, simple_delete1) {
} }
// loop to iterate the data // loop to iterate the data
tdbDbcOpen(pDb, &pDbc, NULL); tdbTbcOpen(pDb, &pDbc, NULL);
ret = tdbDbcMoveToFirst(pDbc); ret = tdbTbcMoveToFirst(pDbc);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
pKey = NULL; pKey = NULL;
pData = NULL; pData = NULL;
for (;;) { for (;;) {
ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData); ret = tdbTbcNext(pDbc, &pKey, &nKey, &pData, &nData);
if (ret < 0) break; if (ret < 0) break;
std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " "; std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " ";
@ -407,20 +407,20 @@ TEST(tdb_test, simple_delete1) {
std::cout << std::endl; std::cout << std::endl;
} }
tdbDbcClose(pDbc); tdbTbcClose(pDbc);
tdbCommit(pEnv, &txn); tdbCommit(pEnv, &txn);
closePool(pPool); closePool(pPool);
tdbClose(pDb); tdbTbClose(pDb);
tdbEnvClose(pEnv); tdbClose(pEnv);
} }
TEST(tdb_test, simple_upsert1) { TEST(tdb_test, simple_upsert1) {
int ret; int ret;
TENV *pEnv; TDB *pEnv;
TDB *pDb; TTB *pDb;
int nData = 100000; int nData = 100000;
char key[64]; char key[64];
char data[64]; char data[64];
@ -431,11 +431,11 @@ TEST(tdb_test, simple_upsert1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// open env // open env
ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); ret = tdbOpen("tdb", 4096, 64, &pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// open database // open database
ret = tdbOpen("db.db", -1, -1, NULL, pEnv, &pDb); ret = tdbTbOpen("db.db", -1, -1, NULL, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
pPool = openPool(); pPool = openPool();
@ -446,7 +446,7 @@ TEST(tdb_test, simple_upsert1) {
for (int iData = 0; iData < nData; iData++) { for (int iData = 0; iData < nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d", iData); sprintf(data, "data%d", iData);
ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
@ -454,7 +454,7 @@ TEST(tdb_test, simple_upsert1) {
for (int iData = 0; iData < nData; iData++) { for (int iData = 0; iData < nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d", iData); sprintf(data, "data%d", iData);
ret = tdbGet(pDb, key, strlen(key), &pData, &nData); ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0);
} }
@ -463,7 +463,7 @@ TEST(tdb_test, simple_upsert1) {
for (int iData = 0; iData < nData; iData++) { for (int iData = 0; iData < nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d-u", iData); sprintf(data, "data%d-u", iData);
ret = tdbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
@ -473,11 +473,11 @@ TEST(tdb_test, simple_upsert1) {
for (int iData = 0; iData < nData; iData++) { for (int iData = 0; iData < nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d-u", iData); sprintf(data, "data%d-u", iData);
ret = tdbGet(pDb, key, strlen(key), &pData, &nData); ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0);
} }
tdbClose(pDb); tdbTbClose(pDb);
tdbEnvClose(pEnv); tdbClose(pEnv);
} }

1
source/libs/transport/src/transSrv.c
View File

@ -264,6 +264,7 @@ static void uvHandleReq(SSrvConn* pConn) {
CONN_SHOULD_RELEASE(pConn, pHead); CONN_SHOULD_RELEASE(pConn, pHead);
STransMsg transMsg; STransMsg transMsg;
memset(&transMsg, 0, sizeof(transMsg));
transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.contLen = transContLenFromMsg(pHead->msgLen);
transMsg.pCont = pHead->content; transMsg.pCont = pHead->content;
transMsg.msgType = pHead->msgType; transMsg.msgType = pHead->msgType;

16
tests/script/tsim/tstream/basic1.sim
View File

@ -24,7 +24,7 @@ sql insert into t1 values(1648791233002,3,2,3,2.1);
sql insert into t1 values(1648791243003,4,2,3,3.1); sql insert into t1 values(1648791243003,4,2,3,3.1);
sql insert into t1 values(1648791213004,4,2,3,4.1); sql insert into t1 values(1648791213004,4,2,3,4.1);
sleep 1000 sleep 1000
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt;
if $rows != 4 then if $rows != 4 then
print ======$rows print ======$rows
@ -137,7 +137,7 @@ endi
sql insert into t1 values(1648791223001,12,14,13,11.1); sql insert into t1 values(1648791223001,12,14,13,11.1);
sleep 500 sleep 500
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt;
if $rows != 4 then if $rows != 4 then
print ======$rows print ======$rows
@ -250,7 +250,7 @@ endi
sql insert into t1 values(1648791223002,12,14,13,11.1); sql insert into t1 values(1648791223002,12,14,13,11.1);
sleep 100 sleep 100
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt;
# row 1 # row 1
if $data11 != 2 then if $data11 != 2 then
@ -280,7 +280,7 @@ endi
sql insert into t1 values(1648791223003,12,14,13,11.1); sql insert into t1 values(1648791223003,12,14,13,11.1);
sleep 100 sleep 100
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt;
# row 1 # row 1
if $data11 != 3 then if $data11 != 3 then
@ -312,7 +312,7 @@ sql insert into t1 values(1648791223001,1,1,1,1.1);
sql insert into t1 values(1648791223002,2,2,2,2.1); sql insert into t1 values(1648791223002,2,2,2,2.1);
sql insert into t1 values(1648791223003,3,3,3,3.1); sql insert into t1 values(1648791223003,3,3,3,3.1);
sleep 100 sleep 100
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt;
# row 1 # row 1
if $data11 != 3 then if $data11 != 3 then
@ -344,7 +344,7 @@ sql insert into t1 values(1648791233003,3,2,3,2.1);
sql insert into t1 values(1648791233002,5,6,7,8.1); sql insert into t1 values(1648791233002,5,6,7,8.1);
sql insert into t1 values(1648791233002,3,2,3,2.1); sql insert into t1 values(1648791233002,3,2,3,2.1);
sleep 100 sleep 100
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt;
# row 2 # row 2
if $data21 != 2 then if $data21 != 2 then
@ -374,7 +374,7 @@ endi
sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1);
sleep 100 sleep 100
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt;
# row 0 # row 0
if $data01 != 4 then if $data01 != 4 then
@ -404,7 +404,7 @@ endi
sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1);
sleep 100 sleep 100
sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt;
# row 1 # row 1
if $data11 != 4 then if $data11 != 4 then

35
tests/system-test/7-tmq/subscribeDb.py
View File

@ -219,6 +219,41 @@ class TDTestCase:
tdSql.query("drop topic %s"%topicName1) tdSql.query("drop topic %s"%topicName1)
tdLog.info("creat the same topic name , and start to consume")
self.initConsumerTable()
tdLog.info("create topics from db")
topicName1 = 'topic_db1'
tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
consumerId = 0
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
topicList = topicName1
ifcheckdata = 0
ifManualCommit = 0
keyList = 'group.id:cgrp1,\
enable.auto.commit:false,\
auto.commit.interval.ms:6000,\
auto.offset.reset:earliest'
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor")
pollDelay = 5
showMsg = 1
showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
expectRows = 1
resultList = self.selectConsumeResult(expectRows)
totalConsumeRows = 0
for i in range(expectRows):
totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
tdLog.exit("tmq consume rows error!")
tdSql.query("drop topic %s"%topicName1)
tdLog.printNoPrefix("======== test case 1 end ...... ") tdLog.printNoPrefix("======== test case 1 end ...... ")
def tmqCase2(self, cfgPath, buildPath): def tmqCase2(self, cfgPath, buildPath):