Merge remote-tracking branch 'origin/3.0' into fix/tsim
This commit is contained in:
Shengliang Guan
commit
32bea46f3c
|
|
@ -118,6 +118,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
|
|||
# ================================================================================================
|
||||
# googletest
|
||||
if(${BUILD_TEST})
|
||||
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
|
||||
add_subdirectory(googletest EXCLUDE_FROM_ALL)
|
||||
target_include_directories(
|
||||
gtest
|
||||
|
|
@ -259,7 +260,7 @@ if(${BUILD_MSVCREGEX})
|
|||
SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
|
||||
endif(${BUILD_MSVCREGEX})
|
||||
|
||||
# msvcregex
|
||||
# wcwidth
|
||||
if(${BUILD_WCWIDTH})
|
||||
add_library(wcwidth STATIC "")
|
||||
target_sources(wcwidth
|
||||
|
|
|
|||
|
|
@ -55,8 +55,9 @@ extern int32_t tMsgDict[];
|
|||
|
||||
#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8)
|
||||
#define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff)
|
||||
#define TMSG_INFO(TYPE) tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)]
|
||||
#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
|
||||
#define TMSG_INFO(TYPE) \
|
||||
(((TYPE) >= 0 && (TYPE) < TDMT_MAX) ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] : 0)
|
||||
#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
|
||||
|
||||
typedef uint16_t tmsg_t;
|
||||
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ typedef struct SScanLogicNode {
|
|||
typedef struct SJoinLogicNode {
|
||||
SLogicNode node;
|
||||
EJoinType joinType;
|
||||
SNode* pMergeCondition;
|
||||
SNode* pOnConditions;
|
||||
bool isSingleTableJoin;
|
||||
} SJoinLogicNode;
|
||||
|
|
@ -329,6 +330,7 @@ typedef struct SInterpFuncPhysiNode {
|
|||
typedef struct SJoinPhysiNode {
|
||||
SPhysiNode node;
|
||||
EJoinType joinType;
|
||||
SNode* pMergeCondition;
|
||||
SNode* pOnConditions;
|
||||
SNodeList* pTargets;
|
||||
} SJoinPhysiNode;
|
||||
|
|
|
|||
|
|
@ -376,6 +376,7 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
|
|||
typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
|
||||
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
|
||||
SNodeList** pCols);
|
||||
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols);
|
||||
|
||||
typedef bool (*FFuncClassifier)(int32_t funcId);
|
||||
int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs);
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
|
|||
SJson *pNodeRetentions = tjsonCreateArray();
|
||||
tjsonAddItemToObject(pJson, "retentions", pNodeRetentions);
|
||||
for (int32_t i = 0; i < nRetention; ++i) {
|
||||
SJson *pNodeRetention = tjsonCreateObject();
|
||||
SJson * pNodeRetention = tjsonCreateObject();
|
||||
const SRetention *pRetention = pCfg->tsdbCfg.retentions + i;
|
||||
tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq);
|
||||
tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit);
|
||||
|
|
@ -118,45 +118,45 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
|
|||
|
||||
int32_t code;
|
||||
tjsonGetNumberValue(pJson, "vgId", pCfg->vgId, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "dbId", pCfg->dbId, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "szPage", pCfg->szPage, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code);
|
||||
if(code < 0) return -1;
|
||||
SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
|
||||
if (code < 0) return -1;
|
||||
SJson * pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
|
||||
int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
|
||||
if (nRetention > TSDB_RETENTION_MAX) {
|
||||
nRetention = TSDB_RETENTION_MAX;
|
||||
|
|
@ -170,30 +170,30 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
|
|||
tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code);
|
||||
}
|
||||
tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
|
||||
tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code);
|
||||
if(code < 0) return -1;
|
||||
if (code < 0) return -1;
|
||||
|
||||
SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
|
||||
int arraySize = tjsonGetArraySize(pNodeInfoArr);
|
||||
|
|
|
|||
|
|
@ -27,10 +27,10 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
|
|||
SMetaReader mer1 = {0};
|
||||
SMetaReader mer2 = {0};
|
||||
char tableFName[TSDB_TABLE_FNAME_LEN];
|
||||
SRpcMsg rpcMsg;
|
||||
SRpcMsg rpcMsg = {0};
|
||||
int32_t code = 0;
|
||||
int32_t rspLen = 0;
|
||||
void *pRsp = NULL;
|
||||
void * pRsp = NULL;
|
||||
SSchemaWrapper schema = {0};
|
||||
SSchemaWrapper schemaTag = {0};
|
||||
|
||||
|
|
@ -111,6 +111,7 @@ _exit:
|
|||
rpcMsg.pCont = pRsp;
|
||||
rpcMsg.contLen = rspLen;
|
||||
rpcMsg.code = code;
|
||||
rpcMsg.msgType = pMsg->msgType;
|
||||
|
||||
if (code) {
|
||||
qError("get table %s meta failed cause of %s", infoReq.tbName, tstrerror(code));
|
||||
|
|
@ -130,10 +131,10 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg) {
|
|||
SMetaReader mer1 = {0};
|
||||
SMetaReader mer2 = {0};
|
||||
char tableFName[TSDB_TABLE_FNAME_LEN];
|
||||
SRpcMsg rpcMsg;
|
||||
SRpcMsg rpcMsg = {0};
|
||||
int32_t code = 0;
|
||||
int32_t rspLen = 0;
|
||||
void *pRsp = NULL;
|
||||
void * pRsp = NULL;
|
||||
SSchemaWrapper schema = {0};
|
||||
SSchemaWrapper schemaTag = {0};
|
||||
|
||||
|
|
@ -220,6 +221,7 @@ _exit:
|
|||
rpcMsg.pCont = pRsp;
|
||||
rpcMsg.contLen = rspLen;
|
||||
rpcMsg.code = code;
|
||||
rpcMsg.msgType = pMsg->msgType;
|
||||
|
||||
if (code) {
|
||||
qError("get table %s cfg failed cause of %s", cfgReq.tbName, tstrerror(code));
|
||||
|
|
@ -260,7 +262,7 @@ void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
|
|||
}
|
||||
|
||||
// wrapper of tsdb read interface
|
||||
tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo* tableList, uint64_t qId,
|
||||
tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *tableList, uint64_t qId,
|
||||
void *pMemRef) {
|
||||
#if 0
|
||||
return tsdbQueryCacheLastT(pVnode->pTsdb, pCond, groupList, qId, pMemRef);
|
||||
|
|
|
|||
|
|
@ -680,7 +680,7 @@ typedef struct SJoinOperatorInfo {
|
|||
SSDataBlock *pRight;
|
||||
int32_t rightPos;
|
||||
SColumnInfo rightCol;
|
||||
SNode *pOnCondition;
|
||||
SNode *pCondAfterMerge;
|
||||
} SJoinOperatorInfo;
|
||||
|
||||
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
|
||||
|
|
|
|||
|
|
@ -53,13 +53,28 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
|
|||
pOperator->info = pInfo;
|
||||
pOperator->pTaskInfo = pTaskInfo;
|
||||
|
||||
SNode* pOnCondition = pJoinNode->pOnConditions;
|
||||
if (nodeType(pOnCondition) == QUERY_NODE_OPERATOR) {
|
||||
SOperatorNode* pNode = (SOperatorNode*)pOnCondition;
|
||||
SNode* pMergeCondition = pJoinNode->pMergeCondition;
|
||||
if (nodeType(pMergeCondition) == QUERY_NODE_OPERATOR) {
|
||||
SOperatorNode* pNode = (SOperatorNode*)pMergeCondition;
|
||||
setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft);
|
||||
setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight);
|
||||
} else if (nodeType(pOnCondition) == QUERY_NODE_LOGIC_CONDITION) {
|
||||
extractTimeCondition(pInfo, (SLogicConditionNode*)pOnCondition);
|
||||
} else {
|
||||
ASSERT(false);
|
||||
}
|
||||
|
||||
if (pJoinNode->pOnConditions != NULL && pJoinNode->node.pConditions != NULL) {
|
||||
pInfo->pCondAfterMerge = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
|
||||
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pInfo->pCondAfterMerge);
|
||||
pLogicCond->pParameterList = nodesMakeList();
|
||||
nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->pOnConditions));
|
||||
nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->node.pConditions));
|
||||
pLogicCond->condType = LOGIC_COND_TYPE_AND;
|
||||
} else if (pJoinNode->pOnConditions != NULL) {
|
||||
pInfo->pCondAfterMerge = nodesCloneNode(pJoinNode->pOnConditions);
|
||||
} else if (pJoinNode->node.pConditions != NULL) {
|
||||
pInfo->pCondAfterMerge = nodesCloneNode(pJoinNode->node.pConditions);
|
||||
} else {
|
||||
pInfo->pCondAfterMerge = NULL;
|
||||
}
|
||||
|
||||
pOperator->fpSet =
|
||||
|
|
@ -88,15 +103,12 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) {
|
|||
|
||||
void destroyMergeJoinOperator(void* param, int32_t numOfOutput) {
|
||||
SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param;
|
||||
nodesDestroyNode(pJoinOperator->pCondAfterMerge);
|
||||
}
|
||||
|
||||
SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
|
||||
static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) {
|
||||
SJoinOperatorInfo* pJoinInfo = pOperator->info;
|
||||
|
||||
SSDataBlock* pRes = pJoinInfo->pRes;
|
||||
blockDataCleanup(pRes);
|
||||
blockDataEnsureCapacity(pRes, 4096);
|
||||
|
||||
int32_t nrows = 0;
|
||||
|
||||
while (1) {
|
||||
|
|
@ -181,7 +193,28 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
|
|||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
|
||||
SJoinOperatorInfo* pJoinInfo = pOperator->info;
|
||||
|
||||
SSDataBlock* pRes = pJoinInfo->pRes;
|
||||
blockDataCleanup(pRes);
|
||||
blockDataEnsureCapacity(pRes, 4096);
|
||||
while (true) {
|
||||
int32_t numOfRowsBefore = pRes->info.rows;
|
||||
doMergeJoinImpl(pOperator, pRes);
|
||||
int32_t numOfNewRows = pRes->info.rows - numOfRowsBefore;
|
||||
if (numOfNewRows == 0) {
|
||||
break;
|
||||
}
|
||||
if (pJoinInfo->pCondAfterMerge != NULL) {
|
||||
doFilter(pJoinInfo->pCondAfterMerge, pRes);
|
||||
}
|
||||
if (pRes->info.rows >= pOperator->resultInfo.threshold) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return (pRes->info.rows > 0) ? pRes : NULL;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1565,6 +1565,10 @@ void constructUdfService(void *argsThread) {
|
|||
//TODO return value of uv_run
|
||||
uv_run(&udfc->uvLoop, UV_RUN_DEFAULT);
|
||||
uv_loop_close(&udfc->uvLoop);
|
||||
|
||||
uv_walk(&udfc->uvLoop, udfUdfdCloseWalkCb, NULL);
|
||||
uv_run(&udfc->uvLoop, UV_RUN_DEFAULT);
|
||||
uv_loop_close(&udfc->uvLoop);
|
||||
}
|
||||
|
||||
int32_t udfcOpen() {
|
||||
|
|
|
|||
|
|
@ -368,6 +368,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
|
|||
static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) {
|
||||
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
|
||||
COPY_SCALAR_FIELD(joinType);
|
||||
CLONE_NODE_FIELD(pMergeCondition);
|
||||
CLONE_NODE_FIELD(pOnConditions);
|
||||
COPY_SCALAR_FIELD(isSingleTableJoin);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
|
|||
|
|
@ -1254,6 +1254,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) {
|
|||
|
||||
static const char* jkJoinLogicPlanJoinType = "JoinType";
|
||||
static const char* jkJoinLogicPlanOnConditions = "OnConditions";
|
||||
static const char* jkJoinLogicPlanMergeCondition = "MergeConditions";
|
||||
|
||||
static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj;
|
||||
|
|
@ -1262,6 +1263,9 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkJoinLogicPlanJoinType, pNode->joinType);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkJoinLogicPlanMergeCondition, nodeToJson, pNode->pMergeCondition);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions);
|
||||
}
|
||||
|
|
@ -1617,6 +1621,7 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
|
|||
}
|
||||
|
||||
static const char* jkJoinPhysiPlanJoinType = "JoinType";
|
||||
static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition";
|
||||
static const char* jkJoinPhysiPlanOnConditions = "OnConditions";
|
||||
static const char* jkJoinPhysiPlanTargets = "Targets";
|
||||
|
||||
|
|
@ -1627,6 +1632,9 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkJoinPhysiPlanMergeCondition, nodeToJson, pNode->pMergeCondition);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pOnConditions);
|
||||
}
|
||||
|
|
@ -1648,6 +1656,9 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeObject(pJson, jkJoinPhysiPlanMergeCondition, &pNode->pMergeCondition);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -470,6 +470,9 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
|
|||
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
|
||||
SJoinPhysiNode* pJoin = (SJoinPhysiNode*)pNode;
|
||||
res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
|
||||
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
|
||||
res = walkPhysiPlan(pJoin->pMergeCondition, order, walker, pContext);
|
||||
}
|
||||
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
|
||||
res = walkPhysiPlan(pJoin->pOnConditions, order, walker, pContext);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -718,6 +718,7 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
case QUERY_NODE_LOGIC_PLAN_JOIN: {
|
||||
SJoinLogicNode* pLogicNode = (SJoinLogicNode*)pNode;
|
||||
destroyLogicNode((SLogicNode*)pLogicNode);
|
||||
nodesDestroyNode(pLogicNode->pMergeCondition);
|
||||
nodesDestroyNode(pLogicNode->pOnConditions);
|
||||
break;
|
||||
}
|
||||
|
|
@ -828,6 +829,7 @@ void nodesDestroyNode(SNode* pNode) {
|
|||
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
|
||||
SJoinPhysiNode* pPhyNode = (SJoinPhysiNode*)pNode;
|
||||
destroyPhysiNode((SPhysiNode*)pPhyNode);
|
||||
nodesDestroyNode(pPhyNode->pMergeCondition);
|
||||
nodesDestroyNode(pPhyNode->pOnConditions);
|
||||
nodesDestroyList(pPhyNode->pTargets);
|
||||
break;
|
||||
|
|
@ -1493,6 +1495,38 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char*
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols) {
|
||||
if (NULL == pCols) {
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
SCollectColumnsCxt cxt = {
|
||||
.errCode = TSDB_CODE_SUCCESS,
|
||||
.pTableAlias = pTableAlias,
|
||||
.collectType = type,
|
||||
.pCols = (NULL == *pCols ? nodesMakeList() : *pCols),
|
||||
.pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)};
|
||||
if (NULL == cxt.pCols || NULL == cxt.pColHash) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
*pCols = NULL;
|
||||
|
||||
nodesWalkExpr(node, collectColumns, &cxt);
|
||||
|
||||
taosHashCleanup(cxt.pColHash);
|
||||
if (TSDB_CODE_SUCCESS != cxt.errCode) {
|
||||
nodesDestroyList(cxt.pCols);
|
||||
return cxt.errCode;
|
||||
}
|
||||
if (LIST_LENGTH(cxt.pCols) > 0) {
|
||||
*pCols = cxt.pCols;
|
||||
} else {
|
||||
nodesDestroyList(cxt.pCols);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
||||
}
|
||||
|
||||
typedef struct SCollectFuncsCxt {
|
||||
int32_t errCode;
|
||||
FFuncClassifier classifier;
|
||||
|
|
|
|||
|
|
@ -480,12 +480,18 @@ static int32_t pushDownCondOptPushCondToProject(SOptimizeContext* pCxt, SProject
|
|||
return pushDownCondOptAppendCond(&pProject->node.pConditions, pCond);
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptPushCondToJoin(SOptimizeContext* pCxt, SJoinLogicNode * pJoin, SNode** pCond) {
|
||||
return pushDownCondOptAppendCond(&pJoin->node.pConditions, pCond);
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) {
|
||||
switch (nodeType(pChild)) {
|
||||
case QUERY_NODE_LOGIC_PLAN_SCAN:
|
||||
return pushDownCondOptPushCondToScan(pCxt, (SScanLogicNode*)pChild, pCond);
|
||||
case QUERY_NODE_LOGIC_PLAN_PROJECT:
|
||||
return pushDownCondOptPushCondToProject(pCxt, (SProjectLogicNode*)pChild, pCond);
|
||||
case QUERY_NODE_LOGIC_PLAN_JOIN:
|
||||
return pushDownCondOptPushCondToJoin(pCxt, (SJoinLogicNode*)pChild, pCond);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -554,13 +560,83 @@ static int32_t pushDownCondOptCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogic
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptPartJoinOnCondLogicCond(SJoinLogicNode* pJoin, SNode** ppMergeCond, SNode** ppOnCond) {
|
||||
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions);
|
||||
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SNodeList* pOnConds = NULL;
|
||||
SNode* pCond = NULL;
|
||||
FOREACH(pCond, pLogicCond->pParameterList) {
|
||||
if (pushDownCondOptIsPriKeyEqualCond(pJoin, pCond)) {
|
||||
*ppMergeCond = nodesCloneNode(pCond);
|
||||
} else {
|
||||
code = nodesListMakeAppend(&pOnConds, nodesCloneNode(pCond));
|
||||
}
|
||||
}
|
||||
|
||||
SNode* pTempOnCond = NULL;
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodesMergeConds(&pTempOnCond, &pOnConds);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != *ppMergeCond) {
|
||||
*ppOnCond = pTempOnCond;
|
||||
nodesDestroyNode(pJoin->pOnConditions);
|
||||
pJoin->pOnConditions = NULL;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
nodesDestroyList(pOnConds);
|
||||
nodesDestroyNode(pTempOnCond);
|
||||
return TSDB_CODE_PLAN_INTERNAL_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptPartJoinOnCond(SJoinLogicNode* pJoin, SNode** ppMergeCond, SNode** ppOnCond) {
|
||||
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) &&
|
||||
LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) {
|
||||
return pushDownCondOptPartJoinOnCondLogicCond(pJoin, ppMergeCond, ppOnCond);
|
||||
}
|
||||
|
||||
if (pushDownCondOptIsPriKeyEqualCond(pJoin, pJoin->pOnConditions)) {
|
||||
*ppMergeCond = nodesCloneNode(pJoin->pOnConditions);
|
||||
*ppOnCond = NULL;
|
||||
nodesDestroyNode(pJoin->pOnConditions);
|
||||
pJoin->pOnConditions = NULL;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
return TSDB_CODE_PLAN_INTERNAL_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
|
||||
int32_t code = pushDownCondOptCheckJoinOnCond(pCxt, pJoin);
|
||||
SNode* pJoinMergeCond = NULL;
|
||||
SNode* pJoinOnCond = NULL;
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = pushDownCondOptPartJoinOnCond(pJoin, &pJoinMergeCond, &pJoinOnCond);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
pJoin->pMergeCondition = pJoinMergeCond;
|
||||
pJoin->pOnConditions = pJoinOnCond;
|
||||
} else {
|
||||
nodesDestroyNode(pJoinMergeCond);
|
||||
nodesDestroyNode(pJoinOnCond);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
|
||||
if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (NULL == pJoin->node.pConditions) {
|
||||
return pushDownCondOptCheckJoinOnCond(pCxt, pJoin);
|
||||
int32_t code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
|
||||
pCxt->optimized = true;
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
SNode* pOnCond = NULL;
|
||||
|
|
@ -579,10 +655,13 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p
|
|||
pushDownCondOptPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
|
||||
pCxt->optimized = true;
|
||||
code = pushDownCondOptCheckJoinOnCond(pCxt, pJoin);
|
||||
} else {
|
||||
nodesDestroyNode(pOnCond);
|
||||
nodesDestroyNode(pLeftChildCond);
|
||||
|
|
@ -720,7 +799,8 @@ static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAg
|
|||
// TODO: remove it after full implementation of pushing down to child
|
||||
if (1 != LIST_LENGTH(pAgg->node.pChildren) ||
|
||||
QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pAgg->node.pChildren, 0)) &&
|
||||
QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(nodesListGetNode(pAgg->node.pChildren, 0))) {
|
||||
QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(nodesListGetNode(pAgg->node.pChildren, 0)) &&
|
||||
QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(nodesListGetNode(pAgg->node.pChildren, 0))) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -612,10 +612,8 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
|
|||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
pJoin->joinType = pJoinLogicNode->joinType;
|
||||
if (NULL != pJoinLogicNode->pOnConditions) {
|
||||
code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions,
|
||||
&pJoin->pOnConditions);
|
||||
}
|
||||
setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pMergeCondition,
|
||||
&pJoin->pMergeCondition);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = setListSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->node.pTargets,
|
||||
&pJoin->pTargets);
|
||||
|
|
@ -623,6 +621,21 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc);
|
||||
}
|
||||
|
||||
SNodeList* condCols = nodesMakeList();
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
|
||||
code = nodesCollectColumnsFromNode(pJoinLogicNode->pOnConditions, NULL, COLLECT_COL_TYPE_ALL, &condCols);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = addDataBlockSlots(pCxt, condCols, pJoin->node.pOutputDataBlockDesc);
|
||||
nodesDestroyList(condCols);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pOnConditions) {
|
||||
code = setNodeSlotId(pCxt, ((SPhysiNode*)pJoin)->pOutputDataBlockDesc->dataBlockId, -1, pJoinLogicNode->pOnConditions,
|
||||
&pJoin->pOnConditions);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,9 +83,9 @@ typedef struct SSyncSnapshotReceiver {
|
|||
|
||||
SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId);
|
||||
void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver);
|
||||
int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg);
|
||||
int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver);
|
||||
bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
|
||||
int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg);
|
||||
int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver);
|
||||
bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
|
||||
|
||||
cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver);
|
||||
char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver);
|
||||
|
|
|
|||
|
|
@ -158,13 +158,13 @@ int32_t syncSetStandby(int64_t rid) {
|
|||
}
|
||||
|
||||
if (pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) {
|
||||
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
|
||||
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
|
||||
terrno = TSDB_CODE_SYN_IS_LEADER;
|
||||
} else {
|
||||
terrno = TSDB_CODE_SYN_STANDBY_NOT_READY;
|
||||
}
|
||||
sError("failed to set standby since it is not follower, state:%s rid:%" PRId64, syncStr(pSyncNode->state), rid);
|
||||
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -620,6 +620,7 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) {
|
|||
|
||||
SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid);
|
||||
if (pSyncNode == NULL) {
|
||||
taosReleaseRef(tsNodeRefId, rid);
|
||||
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
|
||||
return -1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
*/
|
||||
|
||||
#include "syncRespMgr.h"
|
||||
#include "syncRaftEntry.h"
|
||||
#include "syncRaftStore.h"
|
||||
|
||||
SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) {
|
||||
|
|
@ -116,4 +117,59 @@ void syncRespClean(SSyncRespMgr *pObj) {
|
|||
taosThreadMutexUnlock(&(pObj->mutex));
|
||||
}
|
||||
|
||||
void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {}
|
||||
void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {
|
||||
SRespStub *pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, NULL);
|
||||
int cnt = 0;
|
||||
SSyncNode *pSyncNode = pObj->data;
|
||||
|
||||
SArray *delIndexArray = taosArrayInit(0, sizeof(SyncIndex));
|
||||
ASSERT(delIndexArray != NULL);
|
||||
|
||||
while (pStub) {
|
||||
size_t len;
|
||||
void *key = taosHashGetKey(pStub, &len);
|
||||
SyncIndex *pIndex = (SyncIndex *)key;
|
||||
|
||||
int64_t nowMS = taosGetTimestampMs();
|
||||
if (nowMS - pStub->createTime > ttl) {
|
||||
taosArrayPush(delIndexArray, pIndex);
|
||||
cnt++;
|
||||
|
||||
SSyncRaftEntry *pEntry = NULL;
|
||||
int32_t code = 0;
|
||||
if (pSyncNode->pLogStore != NULL) {
|
||||
code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, *pIndex, &pEntry);
|
||||
if (code == 0 && pEntry != NULL) {
|
||||
SFsmCbMeta cbMeta = {0};
|
||||
cbMeta.index = pEntry->index;
|
||||
cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pSyncNode, cbMeta.index);
|
||||
cbMeta.isWeak = pEntry->isWeak;
|
||||
cbMeta.code = TSDB_CODE_SYN_TIMEOUT;
|
||||
cbMeta.state = pSyncNode->state;
|
||||
cbMeta.seqNum = pEntry->seqNum;
|
||||
cbMeta.term = pEntry->term;
|
||||
cbMeta.currentTerm = pSyncNode->pRaftStore->currentTerm;
|
||||
cbMeta.flag = 0;
|
||||
|
||||
SRpcMsg rpcMsg = pStub->rpcMsg;
|
||||
rpcMsg.pCont = rpcMallocCont(pEntry->dataLen);
|
||||
memcpy(rpcMsg.pCont, pEntry->data, pEntry->dataLen);
|
||||
pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta);
|
||||
|
||||
syncEntryDestory(pEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, pStub);
|
||||
}
|
||||
|
||||
int32_t arraySize = taosArrayGetSize(delIndexArray);
|
||||
sDebug("vgId:%d, resp clean by ttl, cnt:%d, array-size:%d", pSyncNode->vgId, cnt, arraySize);
|
||||
|
||||
for (int32_t i = 0; i < arraySize; ++i) {
|
||||
SyncIndex *pIndex = taosArrayGet(delIndexArray, i);
|
||||
taosHashRemove(pObj->pRespHash, pIndex, sizeof(SyncIndex));
|
||||
}
|
||||
taosArrayDestroy(delIndexArray);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,9 +22,11 @@
|
|||
#include "wal.h"
|
||||
|
||||
//----------------------------------
|
||||
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm,
|
||||
SyncSnapshotSend *pBeginMsg);
|
||||
static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg);
|
||||
static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg);
|
||||
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg);
|
||||
static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver);
|
||||
static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg);
|
||||
static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg);
|
||||
|
||||
//----------------------------------
|
||||
SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) {
|
||||
|
|
@ -68,7 +70,9 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
|
|||
// close reader
|
||||
if (pSender->pReader != NULL) {
|
||||
int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
|
||||
ASSERT(ret == 0);
|
||||
if (ret != 0) {
|
||||
syncNodeErrorLog(pSender->pSyncNode, "stop reader error");
|
||||
}
|
||||
pSender->pReader = NULL;
|
||||
}
|
||||
|
||||
|
|
@ -79,7 +83,12 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
|
|||
|
||||
bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; }
|
||||
|
||||
// begin send snapshot by snapshot, pReader
|
||||
// begin send snapshot by param, snapshot, pReader
|
||||
//
|
||||
// action:
|
||||
// 1. assert reader not start
|
||||
// 2. update state
|
||||
// 3. send first snapshot block
|
||||
int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapshotParam, SSnapshot snapshot,
|
||||
void *pReader) {
|
||||
ASSERT(!snapshotSenderIsStart(pSender));
|
||||
|
|
@ -98,7 +107,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapsho
|
|||
|
||||
// update term
|
||||
pSender->term = pSender->pSyncNode->pRaftStore->currentTerm;
|
||||
++(pSender->privateTerm);
|
||||
++(pSender->privateTerm); // increase private term
|
||||
|
||||
// update state
|
||||
pSender->finish = false;
|
||||
|
|
@ -114,9 +123,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapsho
|
|||
|
||||
code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore,
|
||||
pSender->snapshot.lastConfigIndex, &pEntry);
|
||||
if (code == 0) {
|
||||
ASSERT(pEntry != NULL);
|
||||
|
||||
if (code == 0 && pEntry != NULL) {
|
||||
SRpcMsg rpcMsg;
|
||||
syncEntry2OriginalRpc(pEntry, &rpcMsg);
|
||||
|
||||
|
|
@ -207,6 +214,8 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) {
|
|||
pSender->start = false;
|
||||
pSender->finish = finish;
|
||||
|
||||
// do not update term, maybe print
|
||||
|
||||
// event log
|
||||
do {
|
||||
char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender stop");
|
||||
|
|
@ -243,6 +252,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) {
|
|||
pMsg->srcId = pSender->pSyncNode->myRaftId;
|
||||
pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
|
||||
pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm;
|
||||
pMsg->beginIndex = pSender->snapshotParam.start;
|
||||
pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
|
||||
pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
|
||||
pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
|
||||
|
|
@ -281,11 +291,13 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
|
|||
pMsg->srcId = pSender->pSyncNode->myRaftId;
|
||||
pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
|
||||
pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm;
|
||||
pMsg->beginIndex = pSender->snapshotParam.start;
|
||||
pMsg->lastIndex = pSender->snapshot.lastApplyIndex;
|
||||
pMsg->lastTerm = pSender->snapshot.lastApplyTerm;
|
||||
pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex;
|
||||
pMsg->lastConfig = pSender->lastConfig;
|
||||
pMsg->seq = pSender->seq;
|
||||
pMsg->privateTerm = pSender->privateTerm;
|
||||
memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
|
||||
|
||||
// send msg
|
||||
|
|
@ -305,6 +317,12 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
|
||||
ASSERT(pMsg->ack == pSender->seq);
|
||||
pSender->ack = pMsg->ack;
|
||||
++(pSender->seq);
|
||||
}
|
||||
|
||||
cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) {
|
||||
char u64buf[128];
|
||||
cJSON *pRoot = cJSON_CreateObject();
|
||||
|
|
@ -371,10 +389,11 @@ char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) {
|
|||
syncUtilU642Addr(destId.addr, host, sizeof(host), &port);
|
||||
|
||||
snprintf(s, len,
|
||||
"%s {%p laindex:%ld laterm:%lu lcindex:%ld seq:%d ack:%d finish:%d pterm:%lu replica-index:%d %s:%d}", event,
|
||||
pSender, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm,
|
||||
pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, pSender->privateTerm,
|
||||
pSender->replicaIndex, host, port);
|
||||
"%s {%p s-param:%ld e-param:%ld laindex:%ld laterm:%lu lcindex:%ld seq:%d ack:%d finish:%d pterm:%lu "
|
||||
"replica-index:%d %s:%d}",
|
||||
event, pSender, pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex,
|
||||
pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack,
|
||||
pSender->finish, pSender->privateTerm, pSender->replicaIndex, host, port);
|
||||
|
||||
return s;
|
||||
}
|
||||
|
|
@ -429,11 +448,10 @@ bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceive
|
|||
// static do start by privateTerm, pBeginMsg
|
||||
// receive first snapshot data
|
||||
// write first block data
|
||||
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm,
|
||||
SyncSnapshotSend *pBeginMsg) {
|
||||
static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) {
|
||||
// update state
|
||||
pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm;
|
||||
pReceiver->privateTerm = privateTerm;
|
||||
pReceiver->privateTerm = pBeginMsg->privateTerm;
|
||||
pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN;
|
||||
pReceiver->fromId = pBeginMsg->srcId;
|
||||
pReceiver->start = true;
|
||||
|
|
@ -445,7 +463,7 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm p
|
|||
pReceiver->snapshotParam.start = pBeginMsg->beginIndex;
|
||||
pReceiver->snapshotParam.end = pBeginMsg->lastIndex;
|
||||
|
||||
// write data
|
||||
// start writer
|
||||
ASSERT(pReceiver->pWriter == NULL);
|
||||
int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm,
|
||||
&(pReceiver->snapshotParam), &(pReceiver->pWriter));
|
||||
|
|
@ -481,10 +499,10 @@ static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) {
|
|||
|
||||
// if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver
|
||||
// if already start, force close, start again
|
||||
int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) {
|
||||
int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) {
|
||||
if (!snapshotReceiverIsStart(pReceiver)) {
|
||||
// first start
|
||||
snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg);
|
||||
snapshotReceiverDoStart(pReceiver, pBeginMsg);
|
||||
|
||||
} else {
|
||||
// already start
|
||||
|
|
@ -494,12 +512,14 @@ int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm private
|
|||
snapshotReceiverForceStop(pReceiver);
|
||||
|
||||
// start again
|
||||
snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg);
|
||||
snapshotReceiverDoStart(pReceiver, pBeginMsg);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// just set start = false
|
||||
// FpSnapshotStopWrite should not be called, assert writer == NULL
|
||||
int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
|
||||
if (pReceiver->pWriter != NULL) {
|
||||
int32_t ret =
|
||||
|
|
@ -522,6 +542,7 @@ int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
// when recv last snapshot block, apply data into snapshot
|
||||
static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
|
||||
ASSERT(pMsg->seq == SYNC_SNAPSHOT_SEQ_END);
|
||||
|
||||
|
|
@ -550,7 +571,7 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
|
|||
pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex;
|
||||
}
|
||||
|
||||
// stop writer
|
||||
// stop writer, apply data
|
||||
code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true);
|
||||
if (code != 0) {
|
||||
syncNodeErrorLog(pReceiver->pSyncNode, "snapshot stop writer true error");
|
||||
|
|
@ -579,15 +600,20 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
|
|||
return 0;
|
||||
}
|
||||
|
||||
// apply data block
|
||||
// update progress
|
||||
static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
|
||||
ASSERT(pMsg->seq == pReceiver->ack + 1);
|
||||
|
||||
if (pReceiver->pWriter != NULL) {
|
||||
if (pMsg->dataLen > 0) {
|
||||
// apply data block
|
||||
int32_t code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter,
|
||||
pMsg->data, pMsg->dataLen);
|
||||
ASSERT(code == 0);
|
||||
}
|
||||
|
||||
// update progress
|
||||
pReceiver->ack = pMsg->seq;
|
||||
|
||||
// event log
|
||||
|
|
@ -665,14 +691,23 @@ char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event)
|
|||
uint16_t port;
|
||||
syncUtilU642Addr(fromId.addr, host, sizeof(host), &port);
|
||||
|
||||
snprintf(s, len, "%s {%p start:%d ack:%d term:%lu pterm:%lu from:%s:%d laindex:%ld laterm:%lu lcindex:%ld}", event,
|
||||
pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->privateTerm, host, port,
|
||||
pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex);
|
||||
snprintf(s, len,
|
||||
"%s {%p start:%d ack:%d term:%lu pterm:%lu from:%s:%d s-param:%ld e-param:%ld laindex:%ld laterm:%lu "
|
||||
"lcindex:%ld}",
|
||||
event, pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->privateTerm, host, port,
|
||||
pReceiver->snapshotParam.start, pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex,
|
||||
pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex);
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
// receiver do something
|
||||
// receiver on message
|
||||
//
|
||||
// condition 1, recv SYNC_SNAPSHOT_SEQ_BEGIN, start receiver, update privateTerm
|
||||
// condition 2, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig)
|
||||
// condition 3, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close
|
||||
// condition 4, got data, update ack
|
||||
//
|
||||
int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
||||
// get receiver
|
||||
SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
|
||||
|
|
@ -683,11 +718,13 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) {
|
||||
if (pMsg->term == pSyncNode->pRaftStore->currentTerm) {
|
||||
if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) {
|
||||
// condition 1
|
||||
// begin, no data
|
||||
snapshotReceiverStart(pReceiver, pMsg->privateTerm, pMsg);
|
||||
snapshotReceiverStart(pReceiver, pMsg);
|
||||
needRsp = true;
|
||||
|
||||
} else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) {
|
||||
// condition 2
|
||||
// end, finish FSM
|
||||
code = snapshotReceiverFinish(pReceiver, pMsg);
|
||||
if (code == 0) {
|
||||
|
|
@ -697,7 +734,6 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
|
||||
// maybe update lastconfig
|
||||
if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) {
|
||||
// int32_t oldReplicaNum = pSyncNode->replicaNum;
|
||||
SSyncCfg oldSyncCfg = pSyncNode->pRaftCfg->cfg;
|
||||
|
||||
// update new config myIndex
|
||||
|
|
@ -709,11 +745,13 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
}
|
||||
|
||||
} else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) {
|
||||
// condition 3
|
||||
// force close
|
||||
snapshotReceiverForceStop(pReceiver);
|
||||
needRsp = false;
|
||||
|
||||
} else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) {
|
||||
// condition 4
|
||||
// transfering
|
||||
if (pMsg->seq == pReceiver->ack + 1) {
|
||||
snapshotReceiverGotData(pReceiver, pMsg);
|
||||
|
|
@ -752,6 +790,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg);
|
||||
syncSnapshotRspDestroy(pRspMsg);
|
||||
}
|
||||
|
||||
} else {
|
||||
// error log
|
||||
do {
|
||||
|
|
@ -759,6 +798,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
syncNodeErrorLog(pSyncNode, eventLog);
|
||||
taosMemoryFree(eventLog);
|
||||
} while (0);
|
||||
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
// error log
|
||||
|
|
@ -767,19 +808,19 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
|
|||
syncNodeErrorLog(pSyncNode, eventLog);
|
||||
taosMemoryFree(eventLog);
|
||||
} while (0);
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
|
||||
ASSERT(pMsg->ack == pSender->seq);
|
||||
pSender->ack = pMsg->ack;
|
||||
++(pSender->seq);
|
||||
}
|
||||
|
||||
// sender receives ack, set seq = ack + 1, send msg from seq
|
||||
// if ack == SYNC_SNAPSHOT_SEQ_END, stop sender
|
||||
// sender on message
|
||||
//
|
||||
// condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender
|
||||
// condition 2 sender receives ack, set seq = ack + 1, send msg from seq
|
||||
// condition 3 sender receives error msg, just print error log
|
||||
//
|
||||
int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
|
||||
// if already drop replica, do not process
|
||||
if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
|
||||
|
|
@ -794,12 +835,14 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
|
|||
// state, term, seq/ack
|
||||
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
|
||||
if (pMsg->term == pSyncNode->pRaftStore->currentTerm) {
|
||||
// receiver ack is finish, close sender
|
||||
// condition 1
|
||||
// receive ack is finish, close sender
|
||||
if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) {
|
||||
snapshotSenderStop(pSender, true);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// condition 2
|
||||
// send next msg
|
||||
if (pMsg->ack == pSender->seq) {
|
||||
// update sender ack
|
||||
|
|
@ -807,6 +850,7 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) {
|
|||
snapshotSend(pSender);
|
||||
|
||||
} else if (pMsg->ack == pSender->seq - 1) {
|
||||
// maybe resend
|
||||
snapshotReSend(pSender);
|
||||
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,13 @@ void print(SHashObj *pNextIndex) {
|
|||
printf("----------------\n");
|
||||
uint64_t *p = (uint64_t *)taosHashIterate(pNextIndex, NULL);
|
||||
while (p) {
|
||||
printf("%lu \n", *p);
|
||||
|
||||
size_t len;
|
||||
void* key = taosHashGetKey(p, &len);
|
||||
|
||||
SRaftId *pRaftId = (SRaftId*)key;
|
||||
|
||||
printf("key:<%lu, %d>, value:%lu \n", pRaftId->addr, pRaftId->vgId, *p);
|
||||
p = (uint64_t *)taosHashIterate(pNextIndex, p);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,9 +73,15 @@ void syncRespMgrGetAndDelTest(uint64_t i) {
|
|||
}
|
||||
}
|
||||
|
||||
SSyncNode *createSyncNode() {
|
||||
SSyncNode *pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode));
|
||||
memset(pSyncNode, 0, sizeof(SSyncNode));
|
||||
return pSyncNode;
|
||||
}
|
||||
|
||||
void test1() {
|
||||
printf("------- test1 ---------\n");
|
||||
pMgr = syncRespMgrCreate(NULL, 0);
|
||||
pMgr = syncRespMgrCreate(createSyncNode(), 0);
|
||||
assert(pMgr != NULL);
|
||||
|
||||
syncRespMgrInsert(10);
|
||||
|
|
@ -100,7 +106,7 @@ void test1() {
|
|||
|
||||
void test2() {
|
||||
printf("------- test2 ---------\n");
|
||||
pMgr = syncRespMgrCreate(NULL, 0);
|
||||
pMgr = syncRespMgrCreate(createSyncNode(), 0);
|
||||
assert(pMgr != NULL);
|
||||
|
||||
syncRespMgrInsert(10);
|
||||
|
|
@ -117,7 +123,7 @@ void test2() {
|
|||
|
||||
void test3() {
|
||||
printf("------- test3 ---------\n");
|
||||
pMgr = syncRespMgrCreate(NULL, 0);
|
||||
pMgr = syncRespMgrCreate(createSyncNode(), 0);
|
||||
assert(pMgr != NULL);
|
||||
|
||||
syncRespMgrInsert(10);
|
||||
|
|
@ -132,13 +138,34 @@ void test3() {
|
|||
syncRespMgrDestroy(pMgr);
|
||||
}
|
||||
|
||||
void test4() {
|
||||
printf("------- test4 ---------\n");
|
||||
pMgr = syncRespMgrCreate(createSyncNode(), 2);
|
||||
assert(pMgr != NULL);
|
||||
|
||||
syncRespMgrInsert(5);
|
||||
syncRespMgrPrint();
|
||||
|
||||
taosMsleep(3000);
|
||||
|
||||
syncRespMgrInsert(3);
|
||||
syncRespMgrPrint();
|
||||
|
||||
printf("====== after clean ttl \n");
|
||||
syncRespClean(pMgr);
|
||||
syncRespMgrPrint();
|
||||
|
||||
syncRespMgrDestroy(pMgr);
|
||||
}
|
||||
|
||||
int main() {
|
||||
tsAsyncLog = 0;
|
||||
sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
|
||||
sDebugFlag = DEBUG_DEBUG + DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
|
||||
logTest();
|
||||
test1();
|
||||
test2();
|
||||
test3();
|
||||
test4();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,8 +40,8 @@ struct SBTree {
|
|||
#define TDB_BTREE_PAGE_IS_ROOT(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_ROOT)
|
||||
#define TDB_BTREE_PAGE_IS_LEAF(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_LEAF)
|
||||
#define TDB_BTREE_PAGE_IS_OVFL(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_OVFL)
|
||||
#define TDB_BTREE_ASSERT_FLAG(flags) \
|
||||
ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \
|
||||
#define TDB_BTREE_ASSERT_FLAG(flags) \
|
||||
ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \
|
||||
TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0) || \
|
||||
TDB_FLAG_IS(flags, TDB_BTREE_OVFL))
|
||||
|
||||
|
|
@ -58,7 +58,7 @@ typedef struct {
|
|||
|
||||
static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2);
|
||||
static int tdbBtreeOpenImpl(SBTree *pBt);
|
||||
//static int tdbBtreeInitPage(SPage *pPage, void *arg, int init);
|
||||
// static int tdbBtreeInitPage(SPage *pPage, void *arg, int init);
|
||||
static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell,
|
||||
int *szCell, TXN *pTxn, SBTree *pBt);
|
||||
static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder, TXN *pTxn, SBTree *pBt);
|
||||
|
|
@ -321,7 +321,7 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
|
|||
|
||||
{
|
||||
// 1. TODO: Search the main DB to check if the DB exists
|
||||
ret = tdbPagerOpenDB(pBt->pPager, &pgno, true);
|
||||
ret = tdbPagerOpenDB(pBt->pPager, &pgno, true, pBt);
|
||||
ASSERT(ret == 0);
|
||||
}
|
||||
|
||||
|
|
@ -721,7 +721,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
|
|||
int szNewCell;
|
||||
SPgno pgno;
|
||||
pgno = TDB_PAGE_PGNO(pNews[iNew]);
|
||||
tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell, pTxn, pBt);
|
||||
tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell, pTxn,
|
||||
pBt);
|
||||
tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0);
|
||||
tdbOsFree(pNewCell);
|
||||
}
|
||||
|
|
@ -916,10 +917,10 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
|
|||
int surplus = minLocal + (nPayload + nHeader - minLocal) % (maxLocal - sizeof(SPgno));
|
||||
int nLocal = surplus <= maxLocal ? surplus : minLocal;
|
||||
|
||||
//int ofpCap = tdbPageCapacity(pBt->pageSize, sizeof(SIntHdr));
|
||||
// int ofpCap = tdbPageCapacity(pBt->pageSize, sizeof(SIntHdr));
|
||||
|
||||
// fetch a new ofp and make it dirty
|
||||
SPgno pgno = 0;
|
||||
SPgno pgno = 0;
|
||||
SPage *ofp, *nextOfp;
|
||||
|
||||
ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt);
|
||||
|
|
@ -942,8 +943,8 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
|
|||
nLeft -= kLen;
|
||||
// pack partial val to local if any space left
|
||||
if (nLocal > kLen + 4) {
|
||||
memcpy(pCell + nHeader + kLen, pVal, nLocal - kLen - sizeof(SPgno));
|
||||
nLeft -= nLocal - kLen - sizeof(SPgno);
|
||||
memcpy(pCell + nHeader + kLen, pVal, nLocal - kLen - sizeof(SPgno));
|
||||
nLeft -= nLocal - kLen - sizeof(SPgno);
|
||||
}
|
||||
|
||||
// pack nextPgno
|
||||
|
|
@ -951,132 +952,132 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
|
|||
|
||||
// pack left val data to ovpages
|
||||
do {
|
||||
lastPage = 0;
|
||||
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
lastPage = 1;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
lastPage = 0;
|
||||
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
lastPage = 1;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
|
||||
// fetch next ofp if not last page
|
||||
if (!lastPage) {
|
||||
// fetch a new ofp and make it dirty
|
||||
ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
tdbFree(pBuf);
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
pgno = 0;
|
||||
}
|
||||
// fetch next ofp if not last page
|
||||
if (!lastPage) {
|
||||
// fetch a new ofp and make it dirty
|
||||
ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
tdbFree(pBuf);
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
pgno = 0;
|
||||
}
|
||||
|
||||
memcpy(pBuf, ((SCell *)pVal) + vLen - nLeft, bytes);
|
||||
memcpy(pBuf + bytes, &pgno, sizeof(pgno));
|
||||
memcpy(pBuf, ((SCell *)pVal) + vLen - nLeft, bytes);
|
||||
memcpy(pBuf + bytes, &pgno, sizeof(pgno));
|
||||
|
||||
ret = tdbPageInsertCell(ofp, 0, pBuf, bytes + sizeof(pgno), 0);
|
||||
if (ret < 0) {
|
||||
tdbFree(pBuf);
|
||||
return -1;
|
||||
}
|
||||
ret = tdbPageInsertCell(ofp, 0, pBuf, bytes + sizeof(pgno), 0);
|
||||
if (ret < 0) {
|
||||
tdbFree(pBuf);
|
||||
return -1;
|
||||
}
|
||||
|
||||
ofp = nextOfp;
|
||||
nLeft -= bytes;
|
||||
ofp = nextOfp;
|
||||
nLeft -= bytes;
|
||||
} while (nLeft > 0);
|
||||
} else {
|
||||
int nLeftKey = kLen;
|
||||
// pack partial key and nextPgno
|
||||
memcpy(pCell + nHeader, pKey, nLocal - 4);
|
||||
nLeft -= nLocal - 4;
|
||||
nLeftKey -= nLocal -4;
|
||||
nLeftKey -= nLocal - 4;
|
||||
|
||||
memcpy(pCell + nHeader + nLocal - 4, &pgno, sizeof(pgno));
|
||||
|
||||
int lastKeyPageSpace = 0;
|
||||
// pack left key & val to ovpages
|
||||
do {
|
||||
// cal key to cpy
|
||||
int lastKeyPage = 0;
|
||||
if (nLeftKey <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeftKey;
|
||||
lastKeyPage = 1;
|
||||
lastKeyPageSpace = ofp->maxLocal - sizeof(SPgno) - nLeftKey;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
// cal key to cpy
|
||||
int lastKeyPage = 0;
|
||||
if (nLeftKey <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeftKey;
|
||||
lastKeyPage = 1;
|
||||
lastKeyPageSpace = ofp->maxLocal - sizeof(SPgno) - nLeftKey;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
|
||||
// cpy key
|
||||
memcpy(pBuf, ((SCell *)pKey) + kLen - nLeftKey, bytes);
|
||||
// cpy key
|
||||
memcpy(pBuf, ((SCell *)pKey) + kLen - nLeftKey, bytes);
|
||||
|
||||
if (lastKeyPage) {
|
||||
if (lastKeyPageSpace >= vLen) {
|
||||
memcpy(pBuf + kLen -nLeftKey, pVal, vLen);
|
||||
if (lastKeyPage) {
|
||||
if (lastKeyPageSpace >= vLen) {
|
||||
memcpy(pBuf + kLen - nLeftKey, pVal, vLen);
|
||||
|
||||
nLeft -= vLen;
|
||||
pgno = 0;
|
||||
} else {
|
||||
memcpy(pBuf + kLen -nLeftKey, pVal, lastKeyPageSpace);
|
||||
nLeft -= lastKeyPageSpace;
|
||||
nLeft -= vLen;
|
||||
pgno = 0;
|
||||
} else {
|
||||
memcpy(pBuf + kLen - nLeftKey, pVal, lastKeyPageSpace);
|
||||
nLeft -= lastKeyPageSpace;
|
||||
|
||||
// fetch next ofp, a new ofp and make it dirty
|
||||
ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// fetch next ofp, a new ofp and make it dirty
|
||||
ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
// fetch next ofp, a new ofp and make it dirty
|
||||
ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// fetch next ofp, a new ofp and make it dirty
|
||||
ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(pBuf + kLen - nLeft, &pgno, sizeof(pgno));
|
||||
memcpy(pBuf + kLen - nLeft, &pgno, sizeof(pgno));
|
||||
|
||||
ret = tdbPageInsertCell(ofp, 0, pBuf, bytes + sizeof(pgno), 0);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
ret = tdbPageInsertCell(ofp, 0, pBuf, bytes + sizeof(pgno), 0);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
ofp = nextOfp;
|
||||
nLeftKey -= bytes;
|
||||
nLeft -= bytes;
|
||||
ofp = nextOfp;
|
||||
nLeftKey -= bytes;
|
||||
nLeft -= bytes;
|
||||
} while (nLeftKey > 0);
|
||||
|
||||
while (nLeft > 0) {
|
||||
// pack left val data to ovpages
|
||||
lastPage = 0;
|
||||
if (nLeft <= maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
lastPage = 1;
|
||||
} else {
|
||||
bytes = maxLocal - sizeof(SPgno);
|
||||
}
|
||||
// pack left val data to ovpages
|
||||
lastPage = 0;
|
||||
if (nLeft <= maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
lastPage = 1;
|
||||
} else {
|
||||
bytes = maxLocal - sizeof(SPgno);
|
||||
}
|
||||
|
||||
// fetch next ofp if not last page
|
||||
if (!lastPage) {
|
||||
// fetch a new ofp and make it dirty
|
||||
ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
tdbFree(pBuf);
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
pgno = 0;
|
||||
}
|
||||
// fetch next ofp if not last page
|
||||
if (!lastPage) {
|
||||
// fetch a new ofp and make it dirty
|
||||
ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
tdbFree(pBuf);
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
pgno = 0;
|
||||
}
|
||||
|
||||
memcpy(pBuf, ((SCell *)pVal) + vLen - nLeft, bytes);
|
||||
memcpy(pBuf + bytes, &pgno, sizeof(pgno));
|
||||
memcpy(pBuf, ((SCell *)pVal) + vLen - nLeft, bytes);
|
||||
memcpy(pBuf + bytes, &pgno, sizeof(pgno));
|
||||
|
||||
ret = tdbPageInsertCell(ofp, 0, pBuf, bytes + sizeof(pgno), 0);
|
||||
if (ret < 0) {
|
||||
tdbFree(pBuf);
|
||||
return -1;
|
||||
}
|
||||
ret = tdbPageInsertCell(ofp, 0, pBuf, bytes + sizeof(pgno), 0);
|
||||
if (ret < 0) {
|
||||
tdbFree(pBuf);
|
||||
return -1;
|
||||
}
|
||||
|
||||
ofp = nextOfp;
|
||||
nLeft -= bytes;
|
||||
ofp = nextOfp;
|
||||
nLeft -= bytes;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1142,7 +1143,8 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, SCellDecoder *pDecoder, TXN *pTxn, SBTree *pBt) {
|
||||
static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, SCellDecoder *pDecoder, TXN *pTxn,
|
||||
SBTree *pBt) {
|
||||
int ret = 0;
|
||||
int nPayload;
|
||||
int maxLocal = pPage->maxLocal;
|
||||
|
|
@ -1171,149 +1173,149 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
|
|||
int surplus = minLocal + (nPayload + nHeader - minLocal) % (maxLocal - sizeof(SPgno));
|
||||
int nLocal = surplus <= maxLocal ? surplus : minLocal;
|
||||
|
||||
int nLeft = nPayload;
|
||||
SPgno pgno = 0;
|
||||
int nLeft = nPayload;
|
||||
SPgno pgno = 0;
|
||||
SPage *ofp;
|
||||
SCell *ofpCell;
|
||||
int bytes;
|
||||
int lastPage = 0;
|
||||
int bytes;
|
||||
int lastPage = 0;
|
||||
|
||||
if (nLocal >= pDecoder->kLen + 4) {
|
||||
pDecoder->pKey = (SCell *)pCell + nHeader;
|
||||
nLeft -= kLen;
|
||||
if (nLocal > kLen + 4) {
|
||||
// read partial val to local
|
||||
pDecoder->pVal = tdbRealloc(pDecoder->pVal, vLen);
|
||||
if (pDecoder->pVal == NULL) {
|
||||
return -1;
|
||||
}
|
||||
TDB_CELLDECODER_SET_FREE_VAL(pDecoder);
|
||||
// read partial val to local
|
||||
pDecoder->pVal = tdbRealloc(pDecoder->pVal, vLen);
|
||||
if (pDecoder->pVal == NULL) {
|
||||
return -1;
|
||||
}
|
||||
TDB_CELLDECODER_SET_FREE_VAL(pDecoder);
|
||||
|
||||
memcpy(pDecoder->pVal, pCell + nHeader + kLen, nLocal - kLen - sizeof(SPgno));
|
||||
memcpy(pDecoder->pVal, pCell + nHeader + kLen, nLocal - kLen - sizeof(SPgno));
|
||||
|
||||
nLeft -= nLocal - kLen - sizeof(SPgno);
|
||||
nLeft -= nLocal - kLen - sizeof(SPgno);
|
||||
}
|
||||
|
||||
memcpy(&pgno, pCell + nHeader + nPayload - nLeft, sizeof(pgno));
|
||||
|
||||
// unpack left val data from ovpages
|
||||
while (pgno != 0) {
|
||||
ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
while (pgno != 0) {
|
||||
ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
ofpCell = tdbPageGetCell(ofp, 0);
|
||||
ofpCell = tdbPageGetCell(ofp, 0);
|
||||
|
||||
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
lastPage = 1;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
lastPage = 1;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
|
||||
memcpy(pDecoder->pVal + vLen - nLeft, ofpCell, bytes);
|
||||
nLeft -= bytes;
|
||||
memcpy(pDecoder->pVal + vLen - nLeft, ofpCell, bytes);
|
||||
nLeft -= bytes;
|
||||
|
||||
memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
|
||||
memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
|
||||
}
|
||||
} else {
|
||||
int nLeftKey = kLen;
|
||||
// load partial key and nextPgno
|
||||
pDecoder->pKey = tdbRealloc(pDecoder->pKey, kLen);
|
||||
if (pDecoder->pKey == NULL) {
|
||||
return -1;
|
||||
return -1;
|
||||
}
|
||||
TDB_CELLDECODER_SET_FREE_KEY(pDecoder);
|
||||
|
||||
memcpy(pDecoder->pKey, pCell + nHeader, nLocal - 4);
|
||||
nLeft -= nLocal - 4;
|
||||
nLeftKey -= nLocal -4;
|
||||
nLeftKey -= nLocal - 4;
|
||||
|
||||
memcpy(&pgno, pCell + nHeader + nLocal - 4, sizeof(pgno));
|
||||
|
||||
int lastKeyPageSpace = 0;
|
||||
// load left key & val to ovpages
|
||||
while (pgno != 0) {
|
||||
ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
ofpCell = tdbPageGetCell(ofp, 0);
|
||||
ofpCell = tdbPageGetCell(ofp, 0);
|
||||
|
||||
int lastKeyPage = 0;
|
||||
if (nLeftKey <= maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeftKey;
|
||||
lastKeyPage = 1;
|
||||
lastKeyPageSpace = ofp->maxLocal - sizeof(SPgno) - nLeftKey;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
int lastKeyPage = 0;
|
||||
if (nLeftKey <= maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeftKey;
|
||||
lastKeyPage = 1;
|
||||
lastKeyPageSpace = ofp->maxLocal - sizeof(SPgno) - nLeftKey;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
|
||||
// cpy key
|
||||
memcpy(pDecoder->pKey + kLen - nLeftKey, ofpCell, bytes);
|
||||
// cpy key
|
||||
memcpy(pDecoder->pKey + kLen - nLeftKey, ofpCell, bytes);
|
||||
|
||||
if (lastKeyPage) {
|
||||
if (lastKeyPageSpace >= vLen) {
|
||||
pDecoder->pVal = ofpCell + kLen -nLeftKey;
|
||||
if (lastKeyPage) {
|
||||
if (lastKeyPageSpace >= vLen) {
|
||||
pDecoder->pVal = ofpCell + kLen - nLeftKey;
|
||||
|
||||
nLeft -= vLen;
|
||||
pgno = 0;
|
||||
} else {
|
||||
// read partial val to local
|
||||
pDecoder->pVal = tdbRealloc(pDecoder->pVal, vLen);
|
||||
if (pDecoder->pVal == NULL) {
|
||||
return -1;
|
||||
}
|
||||
TDB_CELLDECODER_SET_FREE_VAL(pDecoder);
|
||||
nLeft -= vLen;
|
||||
pgno = 0;
|
||||
} else {
|
||||
// read partial val to local
|
||||
pDecoder->pVal = tdbRealloc(pDecoder->pVal, vLen);
|
||||
if (pDecoder->pVal == NULL) {
|
||||
return -1;
|
||||
}
|
||||
TDB_CELLDECODER_SET_FREE_VAL(pDecoder);
|
||||
|
||||
memcpy(pDecoder->pVal, ofpCell + kLen -nLeftKey, lastKeyPageSpace);
|
||||
nLeft -= lastKeyPageSpace;
|
||||
}
|
||||
}
|
||||
memcpy(pDecoder->pVal, ofpCell + kLen - nLeftKey, lastKeyPageSpace);
|
||||
nLeft -= lastKeyPageSpace;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
|
||||
memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
|
||||
|
||||
nLeftKey -= bytes;
|
||||
nLeft -= bytes;
|
||||
nLeftKey -= bytes;
|
||||
nLeft -= bytes;
|
||||
}
|
||||
|
||||
while (nLeft > 0) {
|
||||
ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
ofpCell = tdbPageGetCell(ofp, 0);
|
||||
ofpCell = tdbPageGetCell(ofp, 0);
|
||||
|
||||
// load left val data to ovpages
|
||||
lastPage = 0;
|
||||
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
lastPage = 1;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
// load left val data to ovpages
|
||||
lastPage = 0;
|
||||
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
lastPage = 1;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
|
||||
if (lastPage) {
|
||||
pgno = 0;
|
||||
}
|
||||
if (lastPage) {
|
||||
pgno = 0;
|
||||
}
|
||||
|
||||
if (!pDecoder->pVal) {
|
||||
pDecoder->pVal = tdbRealloc(pDecoder->pVal, vLen);
|
||||
if (pDecoder->pVal == NULL) {
|
||||
return -1;
|
||||
}
|
||||
TDB_CELLDECODER_SET_FREE_VAL(pDecoder);
|
||||
}
|
||||
if (!pDecoder->pVal) {
|
||||
pDecoder->pVal = tdbRealloc(pDecoder->pVal, vLen);
|
||||
if (pDecoder->pVal == NULL) {
|
||||
return -1;
|
||||
}
|
||||
TDB_CELLDECODER_SET_FREE_VAL(pDecoder);
|
||||
}
|
||||
|
||||
memcpy(pDecoder->pVal, ofpCell + vLen - nLeft, bytes);
|
||||
nLeft -= bytes;
|
||||
memcpy(pDecoder->pVal, ofpCell + vLen - nLeft, bytes);
|
||||
nLeft -= bytes;
|
||||
|
||||
memcpy(&pgno, ofpCell + vLen - nLeft + bytes, sizeof(pgno));
|
||||
memcpy(&pgno, ofpCell + vLen - nLeft + bytes, sizeof(pgno));
|
||||
|
||||
nLeft -= bytes;
|
||||
nLeft -= bytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1404,31 +1406,31 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
|
|||
|
||||
// free ofp pages' cells
|
||||
if (dropOfp) {
|
||||
int ret = 0;
|
||||
SPgno pgno = *(SPgno *) (pCell + nHeader + nLocal - sizeof(SPgno));
|
||||
int nLeft = nPayload - nLocal + sizeof(SPgno);
|
||||
int ret = 0;
|
||||
SPgno pgno = *(SPgno *)(pCell + nHeader + nLocal - sizeof(SPgno));
|
||||
int nLeft = nPayload - nLocal + sizeof(SPgno);
|
||||
SPage *ofp;
|
||||
int bytes;
|
||||
int bytes;
|
||||
|
||||
while (pgno != 0) {
|
||||
ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
SCell *ofpCell = tdbPageGetCell(ofp, 0);
|
||||
SCell *ofpCell = tdbPageGetCell(ofp, 0);
|
||||
|
||||
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
|
||||
bytes = nLeft;
|
||||
} else {
|
||||
bytes = ofp->maxLocal - sizeof(SPgno);
|
||||
}
|
||||
|
||||
memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
|
||||
memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
|
||||
|
||||
tdbPagerReturnPage(pPage->pPager, ofp, pTxn);
|
||||
tdbPagerReturnPage(pPage->pPager, ofp, pTxn);
|
||||
|
||||
nLeft -= bytes;
|
||||
nLeft -= bytes;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1932,7 +1934,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int
|
|||
|
||||
// alloc space
|
||||
szBuf = kLen + nData + 14;
|
||||
pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize);
|
||||
pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize);
|
||||
if (pBuf == NULL) {
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ int tdbPagerClose(SPager *pPager) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) {
|
||||
int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) {
|
||||
SPgno pgno;
|
||||
SPage *pPage;
|
||||
int ret;
|
||||
|
|
@ -110,25 +110,41 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) {
|
|||
}
|
||||
|
||||
{
|
||||
// TODO: try to search the main DB to get the page number
|
||||
// pgno = 0;
|
||||
// TODO: try to search the main DB to get the page number
|
||||
// pgno = 0;
|
||||
}
|
||||
|
||||
// if (pgno == 0 && toCreate) {
|
||||
// ret = tdbPagerAllocPage(pPager, &pPage, &pgno);
|
||||
// if (ret < 0) {
|
||||
// return -1;
|
||||
// }
|
||||
if (pgno == 0 && toCreate) {
|
||||
// allocate a new child page
|
||||
TXN txn;
|
||||
tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
|
||||
|
||||
// // TODO: Need to zero the page
|
||||
pPager->inTran = 1;
|
||||
|
||||
// ret = tdbPagerWrite(pPager, pPage);
|
||||
// if (ret < 0) {
|
||||
// return -1;
|
||||
// }
|
||||
// }
|
||||
SBtreeInitPageArg zArg;
|
||||
zArg.flags = 0x1 | 0x2; // root leaf node;
|
||||
zArg.pBt = pBt;
|
||||
ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, &txn);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
*ppgno = pgno;
|
||||
// ret = tdbPagerAllocPage(pPager, &pPage, &pgno);
|
||||
// if (ret < 0) {
|
||||
// return -1;
|
||||
//}
|
||||
|
||||
// TODO: Need to zero the page
|
||||
|
||||
ret = tdbPagerWrite(pPager, pPage);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
tdbTxnClose(&txn);
|
||||
}
|
||||
|
||||
*ppgno = pgno;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -427,9 +443,9 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
|
|||
}
|
||||
|
||||
int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
|
||||
int ret = 0;
|
||||
int ret = 0;
|
||||
SPgno journalSize = 0;
|
||||
u8 *pageBuf = NULL;
|
||||
u8 *pageBuf = NULL;
|
||||
|
||||
tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755);
|
||||
if (jfd == NULL) {
|
||||
|
|
@ -454,7 +470,7 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
|
|||
|
||||
for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) {
|
||||
// read pgno & the page from journal
|
||||
SPgno pgno;
|
||||
SPgno pgno;
|
||||
SPage *pPage;
|
||||
|
||||
int ret = tdbOsRead(jfd, &pgno, sizeof(pgno));
|
||||
|
|
|
|||
|
|
@ -128,13 +128,13 @@ typedef struct SBtInfo {
|
|||
#define TDB_CELLDECODER_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV & TDB_CELLD_F_VAL)
|
||||
|
||||
typedef struct {
|
||||
int kLen;
|
||||
u8 *pKey;
|
||||
int vLen;
|
||||
u8 *pVal;
|
||||
SPgno pgno;
|
||||
u8 *pBuf;
|
||||
u8 freeKV;
|
||||
int kLen;
|
||||
u8 *pKey;
|
||||
int vLen;
|
||||
u8 *pVal;
|
||||
SPgno pgno;
|
||||
u8 *pBuf;
|
||||
u8 freeKV;
|
||||
} SCellDecoder;
|
||||
|
||||
struct SBTC {
|
||||
|
|
@ -184,7 +184,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int
|
|||
|
||||
int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager);
|
||||
int tdbPagerClose(SPager *pPager);
|
||||
int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate);
|
||||
int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt);
|
||||
int tdbPagerWrite(SPager *pPager, SPage *pPage);
|
||||
int tdbPagerBegin(SPager *pPager, TXN *pTxn);
|
||||
int tdbPagerCommit(SPager *pPager, TXN *pTxn);
|
||||
|
|
@ -192,7 +192,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initP
|
|||
TXN *pTxn);
|
||||
void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
|
||||
int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
|
||||
int tdbPagerRestore(SPager *pPager, SBTree *pBt);
|
||||
int tdbPagerRestore(SPager *pPager, SBTree *pBt);
|
||||
|
||||
// tdbPCache.c ====================================
|
||||
#define TDB_PCACHE_PAGE \
|
||||
|
|
@ -314,19 +314,18 @@ static inline int tdbTryLockPage(tdb_spinlock_t *pLock) {
|
|||
#define TDB_TRY_LOCK_PAGE(pPage) tdbTryLockPage(&((pPage)->lock))
|
||||
|
||||
// APIs
|
||||
#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage))
|
||||
#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx)
|
||||
#define TDB_PAGE_FREE_SIZE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage)
|
||||
#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno)
|
||||
#define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell, 0, NULL, NULL) + (pPage)->pPageMethods->szOffset)
|
||||
#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset)
|
||||
#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage))
|
||||
#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx)
|
||||
#define TDB_PAGE_FREE_SIZE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage)
|
||||
#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno)
|
||||
#define TDB_BYTES_CELL_TAKEN(pPage, pCell) \
|
||||
((*(pPage)->xCellSize)(pPage, pCell, 0, NULL, NULL) + (pPage)->pPageMethods->szOffset)
|
||||
#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset)
|
||||
|
||||
int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg);
|
||||
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg);
|
||||
void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int,
|
||||
TXN *, SBTree *pBt));
|
||||
void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int,
|
||||
TXN *, SBTree *pBt));
|
||||
void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
|
||||
void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt));
|
||||
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
|
||||
int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt);
|
||||
int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt);
|
||||
|
|
|
|||
|
|
@ -402,8 +402,9 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
|
|||
} else {
|
||||
pHead->msgType = pMsg->msgType;
|
||||
// set up resp msg type
|
||||
if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead))
|
||||
if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead)) {
|
||||
pHead->msgType = pConn->inType + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -687,8 +687,8 @@ class TDTestCase:
|
|||
# to_json()
|
||||
tdSql.query("select to_json('{\"abc\":123}') from jsons1_1")
|
||||
tdSql.checkRows(2)
|
||||
# tdSql.checkData(0, 0, '{"abc":123}')
|
||||
# tdSql.checkData(1, 0, '{"abc":123}')
|
||||
tdSql.checkData(0, 0, '{"abc":123}')
|
||||
tdSql.checkData(1, 0, '{"abc":123}')
|
||||
tdSql.query("select to_json('null') from jsons1_1")
|
||||
tdSql.checkRows(2)
|
||||
tdSql.checkData(0, 0, 'null')
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ from util.log import *
|
|||
from util.cases import *
|
||||
from util.sql import *
|
||||
import numpy as np
|
||||
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
class TDTestCase:
|
||||
def init(self, conn, logSql):
|
||||
|
|
@ -13,205 +14,209 @@ class TDTestCase:
|
|||
self.rowNum = 10
|
||||
self.ts = 1537146000000 # 2018-9-17 09:00:00.000
|
||||
|
||||
self.ts_str = [
|
||||
'2020-1-1',
|
||||
'2020-2-1 00:00:01',
|
||||
'2020-3-1 00:00:00.001',
|
||||
'2020-4-1 00:00:00.001002',
|
||||
'2020-5-1 00:00:00.001002001'
|
||||
|
||||
]
|
||||
self.db_param_precision = ['ms','us','ns']
|
||||
self.time_unit = ['1w','1d','1h','1m','1s','1a','1u']
|
||||
self.error_unit = ['1b','2w','2d','2h','2m','2s','2a','2u','1c','#1']
|
||||
self.ntbname = 'ntb'
|
||||
self.stbname = 'stb'
|
||||
self.ctbname = 'ctb'
|
||||
def get_ms_timestamp(self,ts_str):
|
||||
_ts_str = ts_str
|
||||
if " " in ts_str:
|
||||
p = ts_str.split(" ")[1]
|
||||
if len(p) > 15 :
|
||||
_ts_str = ts_str[:-3]
|
||||
if ':' in _ts_str and '.' in _ts_str:
|
||||
timestamp = datetime.strptime(_ts_str, "%Y-%m-%d %H:%M:%S.%f")
|
||||
date_time = int(int(time.mktime(timestamp.timetuple()))*1000 + timestamp.microsecond/1000)
|
||||
elif ':' in _ts_str and '.' not in _ts_str:
|
||||
timestamp = datetime.strptime(_ts_str, "%Y-%m-%d %H:%M:%S")
|
||||
date_time = int(int(time.mktime(timestamp.timetuple()))*1000 + timestamp.microsecond/1000)
|
||||
else:
|
||||
timestamp = datetime.strptime(_ts_str, "%Y-%m-%d")
|
||||
date_time = int(int(time.mktime(timestamp.timetuple()))*1000 + timestamp.microsecond/1000)
|
||||
return date_time
|
||||
def get_us_timestamp(self,ts_str):
|
||||
_ts = self.get_ms_timestamp(ts_str) * 1000
|
||||
if " " in ts_str:
|
||||
p = ts_str.split(" ")[1]
|
||||
if len(p) > 12:
|
||||
us_ts = p[12:15]
|
||||
_ts += int(us_ts)
|
||||
return _ts
|
||||
def get_ns_timestamp(self,ts_str):
|
||||
_ts = self.get_us_timestamp(ts_str) *1000
|
||||
if " " in ts_str:
|
||||
p = ts_str.split(" ")[1]
|
||||
if len(p) > 15:
|
||||
us_ts = p[15:]
|
||||
_ts += int(us_ts)
|
||||
return _ts
|
||||
def time_transform(self,ts_str,precision):
|
||||
date_time = []
|
||||
if precision == 'ms':
|
||||
for i in ts_str:
|
||||
date_time.append(self.get_ms_timestamp(i))
|
||||
elif precision == 'us':
|
||||
for i in ts_str:
|
||||
date_time.append(self.get_us_timestamp(i))
|
||||
elif precision == 'ns':
|
||||
for i in ts_str:
|
||||
date_time.append(self.get_us_timestamp(i))
|
||||
return date_time
|
||||
def check_ms_timestamp(self,unit,date_time):
|
||||
if unit.lower() == '1a':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_ms_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]))
|
||||
elif unit.lower() == '1s':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_ms_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000)*1000)
|
||||
elif unit.lower() == '1m':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_ms_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000/60)*60*1000)
|
||||
elif unit.lower() == '1h':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_ms_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60)*60*60*1000 )
|
||||
elif unit.lower() == '1d':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_ms_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24)*24*60*60*1000)
|
||||
elif unit.lower() == '1w':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_ms_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24/7)*7*24*60*60*1000)
|
||||
def check_us_timestamp(self,unit,date_time):
|
||||
if unit.lower() == '1u':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_us_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]))
|
||||
elif unit.lower() == '1a':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_us_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000)*1000)
|
||||
elif unit.lower() == '1s':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_us_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000)*1000*1000)
|
||||
elif unit.lower() == '1m':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_us_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60)*60*1000*1000)
|
||||
elif unit.lower() == '1h':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_us_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60)*60*60*1000*1000 )
|
||||
elif unit.lower() == '1d':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_us_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24)*24*60*60*1000*1000 )
|
||||
elif unit.lower() == '1w':
|
||||
for i in range(len(self.ts_str)):
|
||||
ts_result = self.get_us_timestamp(str(tdSql.queryResult[i][0]))
|
||||
tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24/7)*7*24*60*60*1000*1000)
|
||||
def check_ns_timestamp(self,unit,date_time):
|
||||
if unit.lower() == '1u':
|
||||
for i in range(len(self.ts_str)):
|
||||
tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000)*1000)
|
||||
elif unit.lower() == '1a':
|
||||
for i in range(len(self.ts_str)):
|
||||
tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000)*1000*1000)
|
||||
elif unit.lower() == '1s':
|
||||
for i in range(len(self.ts_str)):
|
||||
tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000)*1000*1000*1000)
|
||||
elif unit.lower() == '1m':
|
||||
for i in range(len(self.ts_str)):
|
||||
tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/60)*60*1000*1000*1000)
|
||||
elif unit.lower() == '1h':
|
||||
for i in range(len(self.ts_str)):
|
||||
tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/60/60)*60*60*1000*1000*1000 )
|
||||
elif unit.lower() == '1d':
|
||||
for i in range(len(self.ts_str)):
|
||||
tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/60/60/24)*24*60*60*1000*1000*1000 )
|
||||
elif unit.lower() == '1w':
|
||||
for i in range(len(self.ts_str)):
|
||||
tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/60/60/24/7)*7*24*60*60*1000*1000*1000)
|
||||
def data_check(self,date_time,precision,tb_type):
|
||||
for unit in self.time_unit:
|
||||
if (unit.lower() == '1u' and precision.lower() == 'ms') or () :
|
||||
if tb_type.lower() == 'ntb':
|
||||
tdSql.error(f'select timetruncate(ts,{unit}) from {self.ntbname}')
|
||||
elif tb_type.lower() == 'ctb':
|
||||
tdSql.error(f'select timetruncate(ts,{unit}) from {self.ctbname}')
|
||||
elif tb_type.lower() == 'stb':
|
||||
tdSql.error(f'select timetruncate(ts,{unit}) from {self.stbname}')
|
||||
elif precision.lower() == 'ms':
|
||||
if tb_type.lower() == 'ntb':
|
||||
tdSql.query(f'select timetruncate(ts,{unit}) from {self.ntbname}')
|
||||
elif tb_type.lower() == 'ctb':
|
||||
tdSql.query(f'select timetruncate(ts,{unit}) from {self.ctbname}')
|
||||
elif tb_type.lower() == 'stb':
|
||||
tdSql.query(f'select timetruncate(ts,{unit}) from {self.stbname}')
|
||||
tdSql.checkRows(len(self.ts_str))
|
||||
self.check_ms_timestamp(unit,date_time)
|
||||
elif precision.lower() == 'us':
|
||||
if tb_type.lower() == 'ntb':
|
||||
tdSql.query(f'select timetruncate(ts,{unit}) from {self.ntbname}')
|
||||
elif tb_type.lower() == 'ctb':
|
||||
tdSql.query(f'select timetruncate(ts,{unit}) from {self.ctbname}')
|
||||
elif tb_type.lower() == 'stb':
|
||||
tdSql.query(f'select timetruncate(ts,{unit}) from {self.stbname}')
|
||||
tdSql.checkRows(len(self.ts_str))
|
||||
self.check_us_timestamp(unit,date_time)
|
||||
elif precision.lower() == 'ns':
|
||||
if tb_type.lower() == 'ntb':
|
||||
tdSql.query(f'select timetruncate(ts,{unit}) from {self.ntbname}')
|
||||
elif tb_type.lower() == 'ctb':
|
||||
tdSql.query(f'select timetruncate(ts,{unit}) from {self.ctbname}')
|
||||
elif tb_type.lower() == 'stb':
|
||||
tdSql.query(f'select timetruncate(ts,{unit}) from {self.stbname}')
|
||||
tdSql.checkRows(len(self.ts_str))
|
||||
self.check_ns_timestamp(unit,date_time)
|
||||
for unit in self.error_unit:
|
||||
if tb_type.lower() == 'ntb':
|
||||
tdSql.error(f'select timetruncate(ts,{unit}) from {self.ntbname}')
|
||||
elif tb_type.lower() == 'ctb':
|
||||
tdSql.error(f'select timetruncate(ts,{unit}) from {self.ctbname}')
|
||||
elif tb_type.lower() == 'stb':
|
||||
tdSql.error(f'select timetruncate(ts,{unit}) from {self.stbname}')
|
||||
def function_check_ntb(self):
|
||||
for precision in self.db_param_precision:
|
||||
tdSql.execute('drop database if exists db')
|
||||
tdSql.execute(f'create database db precision "{precision}"')
|
||||
tdSql.execute('use db')
|
||||
tdSql.execute(f'create table {self.ntbname} (ts timestamp,c0 int)')
|
||||
for ts in self.ts_str:
|
||||
tdSql.execute(f'insert into {self.ntbname} values("{ts}",1)')
|
||||
date_time = self.time_transform(self.ts_str,precision)
|
||||
self.data_check(date_time,precision,'ntb')
|
||||
|
||||
def function_check_stb(self):
|
||||
for precision in self.db_param_precision:
|
||||
tdSql.execute('drop database if exists db')
|
||||
tdSql.execute(f'create database db precision "{precision}"')
|
||||
tdSql.execute('use db')
|
||||
tdSql.execute(f'create table {self.stbname} (ts timestamp,c0 int) tags(t0 int)')
|
||||
tdSql.execute(f'create table {self.ctbname} using {self.stbname} tags(1)')
|
||||
for ts in self.ts_str:
|
||||
tdSql.execute(f'insert into {self.ctbname} values("{ts}",1)')
|
||||
date_time = self.time_transform(self.ts_str,precision)
|
||||
self.data_check(date_time,precision,'ctb')
|
||||
self.data_check(date_time,precision,'stb')
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
|
||||
intData = []
|
||||
floatData = []
|
||||
|
||||
tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double,
|
||||
col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''')
|
||||
tdSql.execute("create table stb_1 using stb tags('beijing')")
|
||||
tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double,
|
||||
col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''')
|
||||
for i in range(self.rowNum):
|
||||
tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)"
|
||||
% (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
|
||||
intData.append(i + 1)
|
||||
floatData.append(i + 0.1)
|
||||
for i in range(self.rowNum):
|
||||
tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)"
|
||||
% (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
|
||||
intData.append(i + 1)
|
||||
floatData.append(i + 0.1)
|
||||
|
||||
tdSql.query("select timetruncate(1,1d) from ntb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.error("select timetruncate(1,1u) from ntb")
|
||||
#tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(1,1a) from ntb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(1,1m) from ntb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(1,1h) from ntb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(ts,1d) from ntb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 08:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1h) from ntb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1m) from ntb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1s) from ntb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1a) from ntb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.checkData(1,0,"2018-09-17 09:00:00.001")
|
||||
tdSql.checkData(2,0,"2018-09-17 09:00:00.002")
|
||||
tdSql.checkData(3,0,"2018-09-17 09:00:00.003")
|
||||
tdSql.checkData(4,0,"2018-09-17 09:00:00.004")
|
||||
tdSql.checkData(5,0,"2018-09-17 09:00:00.005")
|
||||
tdSql.checkData(6,0,"2018-09-17 09:00:00.006")
|
||||
tdSql.checkData(7,0,"2018-09-17 09:00:00.007")
|
||||
tdSql.checkData(8,0,"2018-09-17 09:00:00.008")
|
||||
tdSql.checkData(9,0,"2018-09-17 09:00:00.009")
|
||||
# tdSql.query("select timetruncate(ts,1u) from ntb")
|
||||
# tdSql.checkRows(10)
|
||||
# tdSql.checkData(0,0,"2018-09-17 09:00:00.000000")
|
||||
# tdSql.checkData(1,0,"2018-09-17 09:00:00.001000")
|
||||
# tdSql.checkData(2,0,"2018-09-17 09:00:00.002000")
|
||||
# tdSql.checkData(3,0,"2018-09-17 09:00:00.003000")
|
||||
# tdSql.checkData(4,0,"2018-09-17 09:00:00.004000")
|
||||
# tdSql.checkData(5,0,"2018-09-17 09:00:00.005000")
|
||||
# tdSql.checkData(6,0,"2018-09-17 09:00:00.006000")
|
||||
# tdSql.checkData(7,0,"2018-09-17 09:00:00.007000")
|
||||
# tdSql.checkData(8,0,"2018-09-17 09:00:00.008000")
|
||||
# tdSql.checkData(9,0,"2018-09-17 09:00:00.009000")
|
||||
# tdSql.query("select timetruncate(ts,1b) from ntb")
|
||||
# tdSql.checkRows(10)
|
||||
# tdSql.checkData(0,0,"2018-09-17 09:00:00.000000000")
|
||||
# tdSql.checkData(1,0,"2018-09-17 09:00:00.001000000")
|
||||
# tdSql.checkData(2,0,"2018-09-17 09:00:00.002000000")
|
||||
# tdSql.checkData(3,0,"2018-09-17 09:00:00.003000000")
|
||||
# tdSql.checkData(4,0,"2018-09-17 09:00:00.004000000")
|
||||
# tdSql.checkData(5,0,"2018-09-17 09:00:00.005000000")
|
||||
# tdSql.checkData(6,0,"2018-09-17 09:00:00.006000000")
|
||||
# tdSql.checkData(7,0,"2018-09-17 09:00:00.007000000")
|
||||
# tdSql.checkData(8,0,"2018-09-17 09:00:00.008000000")
|
||||
# tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000")
|
||||
|
||||
|
||||
tdSql.query("select timetruncate(1,1d) from stb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.error("select timetruncate(1,1u) from stb")
|
||||
#tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(1,1a) from stb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(1,1m) from stb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(1,1h) from stb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(ts,1d) from stb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 08:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1h) from stb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1m) from stb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1s) from stb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1a) from stb")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.checkData(1,0,"2018-09-17 09:00:00.001")
|
||||
tdSql.checkData(2,0,"2018-09-17 09:00:00.002")
|
||||
tdSql.checkData(3,0,"2018-09-17 09:00:00.003")
|
||||
tdSql.checkData(4,0,"2018-09-17 09:00:00.004")
|
||||
tdSql.checkData(5,0,"2018-09-17 09:00:00.005")
|
||||
tdSql.checkData(6,0,"2018-09-17 09:00:00.006")
|
||||
tdSql.checkData(7,0,"2018-09-17 09:00:00.007")
|
||||
tdSql.checkData(8,0,"2018-09-17 09:00:00.008")
|
||||
tdSql.checkData(9,0,"2018-09-17 09:00:00.009")
|
||||
# tdSql.query("select timetruncate(ts,1u) from stb")
|
||||
# tdSql.checkRows(10)
|
||||
# tdSql.checkData(0,0,"2018-09-17 09:00:00.000000")
|
||||
# tdSql.checkData(1,0,"2018-09-17 09:00:00.001000")
|
||||
# tdSql.checkData(2,0,"2018-09-17 09:00:00.002000")
|
||||
# tdSql.checkData(3,0,"2018-09-17 09:00:00.003000")
|
||||
# tdSql.checkData(4,0,"2018-09-17 09:00:00.004000")
|
||||
# tdSql.checkData(5,0,"2018-09-17 09:00:00.005000")
|
||||
# tdSql.checkData(6,0,"2018-09-17 09:00:00.006000")
|
||||
# tdSql.checkData(7,0,"2018-09-17 09:00:00.007000")
|
||||
# tdSql.checkData(8,0,"2018-09-17 09:00:00.008000")
|
||||
# tdSql.checkData(9,0,"2018-09-17 09:00:00.009000")
|
||||
# tdSql.query("select timetruncate(ts,1b) from stb")
|
||||
# tdSql.checkRows(10)
|
||||
# tdSql.checkData(0,0,"2018-09-17 09:00:00.000000000")
|
||||
# tdSql.checkData(1,0,"2018-09-17 09:00:00.001000000")
|
||||
# tdSql.checkData(2,0,"2018-09-17 09:00:00.002000000")
|
||||
# tdSql.checkData(3,0,"2018-09-17 09:00:00.003000000")
|
||||
# tdSql.checkData(4,0,"2018-09-17 09:00:00.004000000")
|
||||
# tdSql.checkData(5,0,"2018-09-17 09:00:00.005000000")
|
||||
# tdSql.checkData(6,0,"2018-09-17 09:00:00.006000000")
|
||||
# tdSql.checkData(7,0,"2018-09-17 09:00:00.007000000")
|
||||
# tdSql.checkData(8,0,"2018-09-17 09:00:00.008000000")
|
||||
# tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000")
|
||||
|
||||
tdSql.query("select timetruncate(1,1d) from stb_1")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.error("select timetruncate(1,1u) from stb_1")
|
||||
#tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(1,1a) from stb_1")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(1,1m) from stb_1")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(1,1h) from stb_1")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.query("select timetruncate(ts,1d) from stb_1")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 08:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1h) from stb_1")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1m) from stb_1")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1s) from stb_1")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.query("select timetruncate(ts,1a) from stb_1")
|
||||
tdSql.checkRows(10)
|
||||
tdSql.checkData(0,0,"2018-09-17 09:00:00.000")
|
||||
tdSql.checkData(1,0,"2018-09-17 09:00:00.001")
|
||||
tdSql.checkData(2,0,"2018-09-17 09:00:00.002")
|
||||
tdSql.checkData(3,0,"2018-09-17 09:00:00.003")
|
||||
tdSql.checkData(4,0,"2018-09-17 09:00:00.004")
|
||||
tdSql.checkData(5,0,"2018-09-17 09:00:00.005")
|
||||
tdSql.checkData(6,0,"2018-09-17 09:00:00.006")
|
||||
tdSql.checkData(7,0,"2018-09-17 09:00:00.007")
|
||||
tdSql.checkData(8,0,"2018-09-17 09:00:00.008")
|
||||
tdSql.checkData(9,0,"2018-09-17 09:00:00.009")
|
||||
# tdSql.query("select timetruncate(ts,1u) from stb_1")
|
||||
# tdSql.checkRows(10)
|
||||
# tdSql.checkData(0,0,"2018-09-17 09:00:00.000000")
|
||||
# tdSql.checkData(1,0,"2018-09-17 09:00:00.001000")
|
||||
# tdSql.checkData(2,0,"2018-09-17 09:00:00.002000")
|
||||
# tdSql.checkData(3,0,"2018-09-17 09:00:00.003000")
|
||||
# tdSql.checkData(4,0,"2018-09-17 09:00:00.004000")
|
||||
# tdSql.checkData(5,0,"2018-09-17 09:00:00.005000")
|
||||
# tdSql.checkData(6,0,"2018-09-17 09:00:00.006000")
|
||||
# tdSql.checkData(7,0,"2018-09-17 09:00:00.007000")
|
||||
# tdSql.checkData(8,0,"2018-09-17 09:00:00.008000")
|
||||
# tdSql.checkData(9,0,"2018-09-17 09:00:00.009000")
|
||||
# tdSql.query("select timetruncate(ts,1b) from stb_1")
|
||||
# tdSql.checkRows(10)
|
||||
# tdSql.checkData(0,0,"2018-09-17 09:00:00.000000000")
|
||||
# tdSql.checkData(1,0,"2018-09-17 09:00:00.001000000")
|
||||
# tdSql.checkData(2,0,"2018-09-17 09:00:00.002000000")
|
||||
# tdSql.checkData(3,0,"2018-09-17 09:00:00.003000000")
|
||||
# tdSql.checkData(4,0,"2018-09-17 09:00:00.004000000")
|
||||
# tdSql.checkData(5,0,"2018-09-17 09:00:00.005000000")
|
||||
# tdSql.checkData(6,0,"2018-09-17 09:00:00.006000000")
|
||||
# tdSql.checkData(7,0,"2018-09-17 09:00:00.007000000")
|
||||
# tdSql.checkData(8,0,"2018-09-17 09:00:00.008000000")
|
||||
# tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000")
|
||||
self.function_check_ntb()
|
||||
self.function_check_stb()
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success("%s successfully executed" % __file__)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,264 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
sys.path.append("./7-tmq")
|
||||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 1
|
||||
self.ctbNum = 1
|
||||
self.rowsPerTbl = 100000
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), False)
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 1,
|
||||
'rowsPerTbl': 100000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
tdLog.printNoPrefix("======== test case 1: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 1,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+9379)
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 0
|
||||
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
if expectRowsList[0] != resultList[0]:
|
||||
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
tmqCom.checkFileContent(consumerId, queryString)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 1 end ...... ")
|
||||
|
||||
def tmqCase2(self):
|
||||
tdLog.printNoPrefix("======== test case 2: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 1,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+9379)
|
||||
# queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
totalRowsInserted = expectRowsList[0]
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 1
|
||||
expectrowcnt = math.ceil(totalRowsInserted/3)
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 0")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
firstConsumeRows = resultList[0]
|
||||
|
||||
if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]):
|
||||
tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
# reinit consume info, and start tmq_sim, then check consume result
|
||||
tmqCom.initConsumerTable()
|
||||
consumerId = 2
|
||||
expectrowcnt = math.ceil(totalRowsInserted/3)
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 1")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
secondConsumeRows = resultList[0]
|
||||
|
||||
if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]):
|
||||
tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
# reinit consume info, and start tmq_sim, then check consume result
|
||||
tmqCom.initConsumerTable()
|
||||
consumerId = 3
|
||||
expectrowcnt = math.ceil(totalRowsInserted/3)
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 1")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
thirdConsumeRows = resultList[0]
|
||||
|
||||
if not (totalRowsInserted >= resultList[0]):
|
||||
tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
# total consume
|
||||
actConsumeTotalRows = firstConsumeRows + secondConsumeRows + thirdConsumeRows
|
||||
|
||||
if not (totalRowsInserted == actConsumeTotalRows):
|
||||
tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 2 end ...... ")
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase1()
|
||||
self.tmqCase2()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,245 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
sys.path.append("./7-tmq")
|
||||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 4
|
||||
self.ctbNum = 3000
|
||||
self.rowsPerTbl = 150
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), False)
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 200,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdDnodes.starttaosd(1)
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
tdLog.printNoPrefix("======== test case 1: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+math.ceil(self.rowsPerTbl/5))
|
||||
# queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 0
|
||||
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
if expectRowsList[0] != resultList[0]:
|
||||
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
# tmqCom.checkFileContent(consumerId, queryString)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 1 end ...... ")
|
||||
|
||||
def tmqCase2(self):
|
||||
tdLog.printNoPrefix("======== test case 2: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 20,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 10,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+math.ceil(self.rowsPerTbl/5))
|
||||
# queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
totalRowsInserted = expectRowsList[0]
|
||||
tdLog.info("select result rows: %d"%totalRowsInserted)
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 1
|
||||
expectrowcnt = math.ceil(totalRowsInserted/3)
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 0")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]):
|
||||
tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
firstConsumeRows = resultList[0]
|
||||
|
||||
# reinit consume info, and start tmq_sim, then check consume result
|
||||
tmqCom.initConsumerTable()
|
||||
consumerId = 2
|
||||
expectrowcnt = math.ceil(totalRowsInserted*2/3)
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 1")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
actConsumeTotalRows = firstConsumeRows + resultList[0]
|
||||
|
||||
if not (expectrowcnt >= resultList[0] and totalRowsInserted == actConsumeTotalRows):
|
||||
tdLog.info("act consume rows, first: %d, second: %d "%(firstConsumeRows, resultList[0]))
|
||||
tdLog.info("and sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 2 end ...... ")
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase1()
|
||||
self.tmqCase2()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,242 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
sys.path.append("./7-tmq")
|
||||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 4
|
||||
self.ctbNum = 3000
|
||||
self.rowsPerTbl = 70
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), False)
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 100,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
# tdDnodes.start(1)
|
||||
tdDnodes.starttaosd(1)
|
||||
return
|
||||
|
||||
def tmqCase1(self):
|
||||
tdLog.printNoPrefix("======== test case 1: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 0
|
||||
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
if expectRowsList[0] != resultList[0]:
|
||||
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
# tmqCom.checkFileContent(consumerId, queryString)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 1 end ...... ")
|
||||
|
||||
def tmqCase2(self):
|
||||
tdLog.printNoPrefix("======== test case 2: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
totalRowsInserted = expectRowsList[0]
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 1
|
||||
expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 0")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]):
|
||||
tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
firstConsumeRows = resultList[0]
|
||||
|
||||
# reinit consume info, and start tmq_sim, then check consume result
|
||||
tmqCom.initConsumerTable()
|
||||
consumerId = 2
|
||||
expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 1")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
actConsumeTotalRows = firstConsumeRows + resultList[0]
|
||||
|
||||
if not (expectrowcnt >= resultList[0] and totalRowsInserted == actConsumeTotalRows):
|
||||
tdLog.info("act consume rows, first: %d, second: %d "%(firstConsumeRows, resultList[0]))
|
||||
tdLog.info("and sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 2 end ...... ")
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase1()
|
||||
self.tmqCase2()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,243 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
sys.path.append("./7-tmq")
|
||||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 1
|
||||
self.ctbNum = 1
|
||||
self.rowsPerTbl = 1000000
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), False)
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 1000,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
return
|
||||
|
||||
def tmqCase3(self):
|
||||
tdLog.printNoPrefix("======== test case 3: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 1,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 15,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
# queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+9379)
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
totalRowsInserted = expectRowsList[0]
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 3
|
||||
expectrowcnt = math.ceil(totalRowsInserted/3)
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
consumerId = 4
|
||||
expectrowcnt = math.ceil(totalRowsInserted * 2/3)
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 0")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 2
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
actConsumeTotalRows = resultList[0] + resultList[1]
|
||||
|
||||
if not (totalRowsInserted == actConsumeTotalRows):
|
||||
tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 3 end ...... ")
|
||||
|
||||
def tmqCase4(self):
|
||||
tdLog.printNoPrefix("======== test case 4: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 1,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 25,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+9379)
|
||||
# queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
totalRowsInserted = expectRowsList[0]
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 5
|
||||
expectrowcnt = math.ceil(totalRowsInserted)
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:500, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 0")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
|
||||
tdLog.info("wait commit notify")
|
||||
tmqCom.getStartCommitNotifyFromTmqsim()
|
||||
|
||||
tdLog.info("pkill consume processor")
|
||||
tdCom.killProcessor("tmq_sim")
|
||||
|
||||
# time.sleep(10)
|
||||
|
||||
# reinit consume info, and start tmq_sim, then check consume result
|
||||
tmqCom.initConsumerTable()
|
||||
consumerId = 6
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 1")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
actConsumeTotalRows = resultList[0]
|
||||
|
||||
if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted):
|
||||
tdLog.info("act consume rows: %d"%(actConsumeTotalRows))
|
||||
tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 4 end ...... ")
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase3()
|
||||
self.tmqCase4()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,247 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
sys.path.append("./7-tmq")
|
||||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 4
|
||||
self.ctbNum = 4000
|
||||
self.rowsPerTbl = 150
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), False)
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 200,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
return
|
||||
|
||||
def tmqCase3(self):
|
||||
tdLog.printNoPrefix("======== test case 3: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 10,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+math.ceil(self.rowsPerTbl/5))
|
||||
# queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
totalRowsInserted = expectRowsList[0]
|
||||
tdLog.info("select result rows: %d"%totalRowsInserted)
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 3
|
||||
expectrowcnt = math.ceil(totalRowsInserted/3)
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
consumerId = 4
|
||||
expectrowcnt = math.ceil(totalRowsInserted*2/3)
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 0")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 2
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
actConsumeTotalRows = resultList[0] + resultList[1]
|
||||
|
||||
if not (totalRowsInserted == actConsumeTotalRows):
|
||||
tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 3 end ...... ")
|
||||
|
||||
def tmqCase4(self):
|
||||
tdLog.printNoPrefix("======== test case 4: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 10,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+math.ceil(self.rowsPerTbl/5))
|
||||
# queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
totalRowsInserted = expectRowsList[0]
|
||||
tdLog.info("select result rows: %d"%totalRowsInserted)
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 5
|
||||
expectrowcnt = math.ceil(totalRowsInserted)
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:300, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 0")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
|
||||
tdLog.info("wait commit notify")
|
||||
tmqCom.getStartCommitNotifyFromTmqsim()
|
||||
# tdLog.info("wait start consume notify")
|
||||
# tmqCom.getStartConsumeNotifyFromTmqsim()
|
||||
|
||||
tdLog.info("pkill consume processor")
|
||||
tdCom.killProcessor("tmq_sim")
|
||||
|
||||
# time.sleep(10)
|
||||
|
||||
# reinit consume info, and start tmq_sim, then check consume result
|
||||
tmqCom.initConsumerTable()
|
||||
consumerId = 6
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 1")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
actConsumeTotalRows = resultList[0]
|
||||
|
||||
if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted):
|
||||
tdLog.info("act consume rows: %d"%(actConsumeTotalRows))
|
||||
tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 4 end ...... ")
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase3()
|
||||
self.tmqCase4()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -0,0 +1,241 @@
|
|||
|
||||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
sys.path.append("./7-tmq")
|
||||
from tmqCommon import *
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 4
|
||||
self.ctbNum = 3000
|
||||
self.rowsPerTbl = 70
|
||||
|
||||
def init(self, conn, logSql):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), False)
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 100,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 3,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tmqCom.initConsumerTable()
|
||||
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1)
|
||||
tdLog.info("create stb")
|
||||
tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
|
||||
tdLog.info("create ctb")
|
||||
tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
tdLog.info("insert data")
|
||||
tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
|
||||
ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
|
||||
startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
|
||||
|
||||
tdLog.info("restart taosd to ensure that the data falls into the disk")
|
||||
tdDnodes.stop(1)
|
||||
tdDnodes.start(1)
|
||||
return
|
||||
|
||||
def tmqCase3(self):
|
||||
tdLog.printNoPrefix("======== test case 3: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 10,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
totalRowsInserted = expectRowsList[0]
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 3
|
||||
expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3)
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
consumerId = 4
|
||||
expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3)
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 0")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 2
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
actConsumeTotalRows = resultList[0] + resultList[1]
|
||||
|
||||
if not (totalRowsInserted == actConsumeTotalRows):
|
||||
tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 3 end ...... ")
|
||||
|
||||
def tmqCase4(self):
|
||||
tdLog.printNoPrefix("======== test case 4: ")
|
||||
paraDict = {'dbName': 'dbt',
|
||||
'dropFlag': 1,
|
||||
'event': '',
|
||||
'vgroups': 1,
|
||||
'stbName': 'stb',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
|
||||
'ctbPrefix': 'ctb',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 10,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 10,
|
||||
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
|
||||
'pollDelay': 10,
|
||||
'showMsg': 1,
|
||||
'showRow': 1,
|
||||
'snapshot': 1}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
topicNameList = ['topic1']
|
||||
expectRowsList = []
|
||||
tmqCom.initConsumerTable()
|
||||
|
||||
tdLog.info("create topics from stb with filter")
|
||||
queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
|
||||
# sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
|
||||
sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
|
||||
tdLog.info("create topic sql: %s"%sqlString)
|
||||
tdSql.execute(sqlString)
|
||||
tdSql.query(queryString)
|
||||
expectRowsList.append(tdSql.getRows())
|
||||
totalRowsInserted = expectRowsList[0]
|
||||
|
||||
# init consume info, and start tmq_sim, then check consume result
|
||||
tdLog.info("insert consume info to consume processor")
|
||||
consumerId = 5
|
||||
expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"])
|
||||
topicList = topicNameList[0]
|
||||
ifcheckdata = 1
|
||||
ifManualCommit = 1
|
||||
keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest'
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 0")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
|
||||
tdLog.info("wait commit notify")
|
||||
tmqCom.getStartCommitNotifyFromTmqsim()
|
||||
|
||||
tdLog.info("pkill consume processor")
|
||||
tdCom.killProcessor("tmq_sim")
|
||||
|
||||
# time.sleep(10)
|
||||
|
||||
# reinit consume info, and start tmq_sim, then check consume result
|
||||
tmqCom.initConsumerTable()
|
||||
consumerId = 6
|
||||
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
|
||||
|
||||
tdLog.info("start consume processor 1")
|
||||
tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
|
||||
tdLog.info("wait the consume result")
|
||||
|
||||
expectRows = 1
|
||||
resultList = tmqCom.selectConsumeResult(expectRows)
|
||||
|
||||
actConsumeTotalRows = resultList[0]
|
||||
|
||||
if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted):
|
||||
tdLog.info("act consume rows: %d"%(actConsumeTotalRows))
|
||||
tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted))
|
||||
tdLog.exit("%d tmq consume rows error!"%consumerId)
|
||||
|
||||
time.sleep(10)
|
||||
for i in range(len(topicNameList)):
|
||||
tdSql.query("drop topic %s"%topicNameList[i])
|
||||
|
||||
tdLog.printNoPrefix("======== test case 4 end ...... ")
|
||||
|
||||
def run(self):
|
||||
tdSql.prepare()
|
||||
self.prepareTestEnv()
|
||||
self.tmqCase3()
|
||||
self.tmqCase4()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -165,3 +165,10 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py
|
|||
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py
|
||||
python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py
|
||||
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py
|
||||
python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py
|
||||
python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py
|
||||
python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py
|
||||
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py
|
||||
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
|
||||
python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue