From 49143d079e841195a0e239e54540a724ef145824 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 24 Feb 2022 05:18:41 -0500 Subject: [PATCH 1/5] TD-13495 physical plan refactoring --- include/libs/nodes/nodes.h | 5 +- include/libs/nodes/plannodes.h | 16 +- include/libs/nodes/querynodes.h | 16 +- source/libs/nodes/src/nodesCloneFuncs.c | 10 - source/libs/nodes/src/nodesCodeFuncs.c | 26 -- source/libs/nodes/src/nodesUtilFuncs.c | 6 +- source/libs/planner/src/plannerImpl.c | 352 +++++++++++++++----- source/libs/planner/test/newPlannerTest.cpp | 4 +- source/libs/scalar/inc/filterInt.h | 12 +- source/libs/scalar/src/filter.c | 22 +- source/libs/scalar/src/scalar.c | 10 +- 11 files changed, 322 insertions(+), 157 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index f09ffa8926..d5958e1b9c 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -62,7 +62,6 @@ typedef enum ENodeType { QUERY_NODE_NODE_LIST, QUERY_NODE_FILL, QUERY_NODE_RAW_EXPR, // Only be used in parser module. - QUERY_NODE_COLUMN_REF, QUERY_NODE_TARGET, QUERY_NODE_TUPLE_DESC, QUERY_NODE_SLOT_DESC, @@ -81,7 +80,9 @@ typedef enum ENodeType { // physical plan node QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, - QUERY_NODE_PHYSICAL_PLAN_PROJECT + QUERY_NODE_PHYSICAL_PLAN_PROJECT, + QUERY_NODE_PHYSICAL_PLAN_JOIN, + QUERY_NODE_PHYSICAL_PLAN_AGG } ENodeType; /** diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 40684f2b26..d8896501ad 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -69,8 +69,6 @@ typedef struct SSlotDescNode { ENodeType type; int16_t slotId; SDataType dataType; - int16_t srcTupleId; - int16_t srcSlotId; bool reserve; bool output; } SSlotDescNode; @@ -115,6 +113,20 @@ typedef struct SProjectPhysiNode { SNodeList* pProjections; } SProjectPhysiNode; +typedef struct SJoinPhysiNode { + SPhysiNode node; + EJoinType joinType; + SNode* pOnConditions; // in or out tuple ? + SNodeList* pTargets; +} SJoinPhysiNode; + +typedef struct SAggPhysiNode { + SPhysiNode node; + SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function + SNodeList* pGroupKeys; // SColumnRefNode list + SNodeList* pAggFuncs; +} SAggPhysiNode; + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 4de4095752..59cd672765 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -58,15 +58,17 @@ typedef struct SColumnNode { char tableAlias[TSDB_TABLE_NAME_LEN]; char colName[TSDB_COL_NAME_LEN]; SNode* pProjectRef; -} SColumnNode; - -typedef struct SColumnRefNode { - ENodeType type; - SDataType dataType; int16_t tupleId; int16_t slotId; - int16_t columnId; -} SColumnRefNode; +} SColumnNode; + +// typedef struct SColumnRefNode { +// ENodeType type; +// SDataType dataType; +// int16_t tupleId; +// int16_t slotId; +// int16_t columnId; +// } SColumnRefNode; typedef struct STargetNode { ENodeType type; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 18a316320e..864e13b773 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -142,14 +142,6 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { return (SNode*)pDst; } -static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) { - dataTypeCopy(&pSrc->dataType, &pDst->dataType); - COPY_SCALAR_FIELD(tupleId); - COPY_SCALAR_FIELD(slotId); - COPY_SCALAR_FIELD(columnId); - return (SNode*)pDst; -} - static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) { COPY_SCALAR_FIELD(tupleId); COPY_SCALAR_FIELD(slotId); @@ -183,8 +175,6 @@ SNode* nodesCloneNode(const SNode* pNode) { return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst); case QUERY_NODE_FUNCTION: return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst); - case QUERY_NODE_COLUMN_REF: - return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst); case QUERY_NODE_TARGET: return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst); case QUERY_NODE_REAL_TABLE: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 356deb2f51..92a6126750 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -55,8 +55,6 @@ static char* nodeName(ENodeType type) { return "NodeList"; case QUERY_NODE_FILL: return "Fill"; - case QUERY_NODE_COLUMN_REF: - return "ColumnRef"; case QUERY_NODE_TARGET: return "Target"; case QUERY_NODE_RAW_EXPR: @@ -503,28 +501,6 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { return code; } -static const char* jkColumnRefDataType = "DataType"; -static const char* jkColumnRefTupleId = "TupleId"; -static const char* jkColumnRefSlotId = "SlotId"; -static const char* jkColumnRefColumnId = "ColumnId"; - -static int32_t columnRefNodeToJson(const void* pObj, SJson* pJson) { - const SColumnRefNode* pNode = (const SColumnRefNode*)pObj; - - int32_t code = tjsonAddObject(pJson, jkColumnRefDataType, dataTypeToJson, &pNode->dataType); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkColumnRefTupleId, pNode->tupleId); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkColumnRefSlotId, pNode->slotId); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkColumnRefColumnId, pNode->columnId); - } - - return code; -} - static const char* jkTargetTupleId = "TupleId"; static const char* jkTargetSlotId = "SlotId"; static const char* jkTargetExpr = "Expr"; @@ -646,8 +622,6 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_NODE_LIST: case QUERY_NODE_FILL: - case QUERY_NODE_COLUMN_REF: - return columnRefNodeToJson(pObj, pJson); case QUERY_NODE_TARGET: return targetNodeToJson(pObj, pJson); case QUERY_NODE_RAW_EXPR: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 3598132217..8810f24ef0 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -79,8 +79,6 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SAggLogicNode)); case QUERY_NODE_LOGIC_PLAN_PROJECT: return makeNode(type, sizeof(SProjectLogicNode)); - case QUERY_NODE_COLUMN_REF: - return makeNode(type, sizeof(SColumnRefNode)); case QUERY_NODE_TARGET: return makeNode(type, sizeof(STargetNode)); case QUERY_NODE_TUPLE_DESC: @@ -93,6 +91,10 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(STableScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return makeNode(type, sizeof(SProjectPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + return makeNode(type, sizeof(SJoinPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_AGG: + return makeNode(type, sizeof(SAggPhysiNode)); default: break; } diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c index be570f0b96..d5b5eb1500 100644 --- a/source/libs/planner/src/plannerImpl.c +++ b/source/libs/planner/src/plannerImpl.c @@ -55,6 +55,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SNode* pExpr; int32_t index = 0; FOREACH(pExpr, pCxt->pExprs) { + if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) { + pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0); + } if (nodesEqualNode(pExpr, *pNode)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); CHECK_ALLOC(pCol, DEAL_RES_ERROR); @@ -406,22 +409,13 @@ typedef struct SPhysiPlanContext { static int32_t getSlotKey(SNode* pNode, char* pKey) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - return sprintf(pKey, "%s.%s", ((SColumnNode*)pNode)->tableAlias, ((SColumnNode*)pNode)->colName); - } else { - return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); + SColumnNode* pCol = (SColumnNode*)pNode; + if ('\0' == pCol->tableAlias[0]) { + return sprintf(pKey, "%s", pCol->colName); + } + return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName); } -} - -static SNode* createColumnRef(SNode* pNode, int16_t tupleId, int16_t slotId) { - SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF); - if (NULL == pCol) { - return NULL; - } - pCol->dataType = ((SExprNode*)pNode)->resType; - pCol->tupleId = tupleId; - pCol->slotId = slotId; - pCol->columnId = (QUERY_NODE_COLUMN == nodeType(pNode) ? ((SColumnNode*)pNode)->colId : -1); - return (SNode*)pCol; + return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); } static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) { @@ -429,10 +423,8 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_ CHECK_ALLOC(pSlot, NULL); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; - pSlot->srcTupleId = -1; - pSlot->srcSlotId = -1; pSlot->reserve = false; - pSlot->output = true; + pSlot->output = false; return (SNode*)pSlot; } @@ -443,17 +435,11 @@ static SNode* createTarget(SNode* pNode, int16_t tupleId, int16_t slotId) { } pTarget->tupleId = tupleId; pTarget->slotId = slotId; - pTarget->pExpr = nodesCloneNode(pNode); - if (NULL == pTarget->pExpr) { - nodesDestroyNode((SNode*)pTarget); - return NULL; - } + pTarget->pExpr = pNode; return (SNode*)pTarget; } -static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple, SNodeList** pOutput) { - pTuple->tupleId = pCxt->nextTupleId++; - +static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple) { SHashObj* pHash = NULL; if (NULL == pTuple->pSlots) { pTuple->pSlots = nodesMakeList(); @@ -469,11 +455,8 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId); } - *pOutput = nodesMakeList(); - CHECK_ALLOC(*pOutput, TSDB_CODE_OUT_OF_MEMORY); - SNode* pNode = NULL; - int16_t slotId = 0; + int16_t slotId = taosHashGetSize(pHash); FOREACH(pNode, pList) { SNode* pSlot = createSlotDesc(pCxt, pNode, slotId); CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY); @@ -482,48 +465,50 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes return TSDB_CODE_OUT_OF_MEMORY; } - SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId); - CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); - if (TSDB_CODE_SUCCESS != nodesListAppend(*pOutput, pTarget)) { - nodesDestroyNode(pTarget); - return TSDB_CODE_OUT_OF_MEMORY; - } - SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId }; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = getSlotKey(pNode, name); CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY); + SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId); + CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); + REPLACE_NODE(pTarget); + ++slotId; } return TSDB_CODE_SUCCESS; } -typedef struct STransformCxt { +typedef struct SSetSlotIdCxt { int32_t errCode; - SHashObj* pHash; -} STransformCxt; + SHashObj* pLeftHash; + SHashObj* pRightHash; +} SSetSlotIdCxt; -static EDealRes doTransform(SNode** pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(*pNode)) { - STransformCxt* pCxt = (STransformCxt*)pContext; +static EDealRes doSetSlotId(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) { + SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(*pNode, name); - SSlotIndex* pIndex = taosHashGet(pCxt->pHash, name, len); - if (NULL != pIndex) { - *pNode = createColumnRef(*pNode, pIndex->tupleId, pIndex->slotId); - CHECK_ALLOC(*pNode, DEAL_RES_ERROR); - return DEAL_RES_IGNORE_CHILD; + int32_t len = getSlotKey(pNode, name); + SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len); + if (NULL == pIndex) { + pIndex = taosHashGet(pCxt->pRightHash, name, len); } + // pIndex is definitely not NULL, otherwise it is a bug + ((SColumnNode*)pNode)->tupleId = pIndex->tupleId; + ((SColumnNode*)pNode)->slotId = pIndex->slotId; + CHECK_ALLOC(pNode, DEAL_RES_ERROR); + return DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; } -static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNode* pNode) { +static SNode* setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNode* pNode) { SNode* pRes = nodesCloneNode(pNode); CHECK_ALLOC(pRes, NULL); - STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; - nodesRewriteNode(&pRes, doTransform, &cxt); + SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId), + .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) }; + nodesWalkNode(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); return NULL; @@ -531,11 +516,12 @@ static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SN return pRes; } -static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNodeList* pList) { +static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNodeList* pList) { SNodeList* pRes = nodesCloneList(pList); CHECK_ALLOC(pRes, NULL); - STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; - nodesRewriteList(pRes, doTransform, &cxt); + SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId), + .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) }; + nodesWalkList(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); return NULL; @@ -543,22 +529,48 @@ static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tup return pRes; } -static SPhysiNode* makePhysiNode(ENodeType type) { +static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); if (NULL == pPhysiNode) { return NULL; } + pPhysiNode->outputTuple.tupleId = pCxt->nextTupleId++; pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC; return pPhysiNode; } -static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { - CHECK_CODE(addTupleDesc(pCxt, pScanLogicNode->pScanCols, &pScanPhysiNode->node.outputTuple, &pScanPhysiNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY); - - if (NULL != pScanLogicNode->node.pConditions) { - pScanPhysiNode->node.pConditions = transformForPhysiPlan(pCxt, pScanPhysiNode->node.outputTuple.tupleId, pScanLogicNode->node.pConditions); - CHECK_ALLOC(pScanPhysiNode->node.pConditions, TSDB_CODE_OUT_OF_MEMORY); +static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) { + if (NULL != pLogicNode->pConditions) { + pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->outputTuple.tupleId, -1, pLogicNode->pConditions); + CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY); } + return TSDB_CODE_SUCCESS; +} + +static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, STupleDescNode* pTuple) { + SHashObj* pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId); + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + SNode* pNode; + FOREACH(pNode, pTargets) { + int32_t len = getSlotKey(pNode, name); + SSlotIndex* pIndex = taosHashGet(pHash, name, len); + ((SSlotDescNode*)nodesListGetNode(pTuple->pSlots, pIndex->slotId))->output = true; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { + if (NULL != pScanLogicNode->pScanCols) { + pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols); + CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); + } + // Tuple describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t + CHECK_CODE(addTupleDesc(pCxt, pScanPhysiNode->pScanCols, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY); + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY); + + CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY); pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; @@ -570,14 +582,14 @@ static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanL } static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { - STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); + STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); CHECK_ALLOC(pTagScan, NULL); CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan); return (SPhysiNode*)pTagScan; } static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { - STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); CHECK_ALLOC(pTableScan, NULL); CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); pTableScan->scanFlag = pScanLogicNode->scanFlag; @@ -597,35 +609,205 @@ static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* default: break; } + return NULL; } -static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SProjectLogicNode* pProjectLogicNode) { - SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_PROJECT); +static SNodeList* createJoinOutputCols(SPhysiPlanContext* pCxt, STupleDescNode* pLeftTuple, STupleDescNode* pRightTuple) { + SNodeList* pCols = nodesMakeList(); + CHECK_ALLOC(pCols, NULL); + SNode* pNode; + FOREACH(pNode, pLeftTuple->pSlots) { + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + goto error; + } + pCol->node.resType = pSlot->dataType; + pCol->tupleId = pLeftTuple->tupleId; + pCol->slotId = pSlot->slotId; + pCol->colId = -1; + if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { + goto error; + } + } + FOREACH(pNode, pRightTuple->pSlots) { + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + goto error; + } + pCol->node.resType = pSlot->dataType; + pCol->tupleId = pRightTuple->tupleId; + pCol->slotId = pSlot->slotId; + pCol->colId = -1; + if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { + goto error; + } + } + return pCols; +error: + nodesDestroyList(pCols); + return NULL; +} + +static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode) { + SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN); + CHECK_ALLOC(pJoin, NULL); + + STupleDescNode* pLeftTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple; + STupleDescNode* pRightTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 1))->outputTuple; + pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftTuple->tupleId, pRightTuple->tupleId, pJoinLogicNode->pOnConditions); + CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin); + + pJoin->pTargets = createJoinOutputCols(pCxt, pLeftTuple, pRightTuple); + CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin); + CHECK_CODE(addTupleDesc(pCxt, pJoin->pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin); + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin); + + CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin); + + return (SPhysiNode*)pJoin; +} + +typedef struct SRewritePrecalcExprsCxt { + int32_t errCode; + int32_t planNodeId; + int32_t rewriteId; + SNodeList* pPrecalcExprs; +} SRewritePrecalcExprsCxt; + +static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { + SNode* pExpr = nodesCloneNode(*pNode); + CHECK_ALLOC(pExpr, DEAL_RES_ERROR); + if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) { + nodesDestroyNode(pExpr); + return DEAL_RES_ERROR; + } + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + nodesDestroyNode(pExpr); + return DEAL_RES_ERROR; + } + SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); + pCol->node.resType = pToBeRewrittenExpr->resType; + strcpy(pCol->colName, pToBeRewrittenExpr->aliasName); + nodesDestroyNode(*pNode); + *pNode = (SNode*)pCol; + return DEAL_RES_IGNORE_CHILD; +} + +static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) { + SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext; + switch (nodeType(*pNode)) { + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: { + return collectAndRewrite(pContext, pNode); + } + case QUERY_NODE_FUNCTION: { + if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) { + return collectAndRewrite(pContext, pNode); + } + } + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SNodeList** pPrecalcExprs, SNodeList** pRewrittenList) { + if (NULL == pList) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == *pPrecalcExprs) { + *pPrecalcExprs = nodesMakeList(); + CHECK_ALLOC(*pPrecalcExprs, TSDB_CODE_OUT_OF_MEMORY); + } + if (NULL == *pRewrittenList) { + *pRewrittenList = nodesMakeList(); + CHECK_ALLOC(*pRewrittenList, TSDB_CODE_OUT_OF_MEMORY); + } + SNode* pNode = NULL; + FOREACH(pNode, pList) { + SNode* pNew = NULL; + if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) { + pNew = nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0)); + } else { + pNew = nodesCloneNode(pNode); + } + CHECK_ALLOC(pNew, TSDB_CODE_OUT_OF_MEMORY); + CHECK_CODE(nodesListAppend(*pRewrittenList, pNew), TSDB_CODE_OUT_OF_MEMORY); + } + SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; + nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); + if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { + nodesDestroyList(cxt.pPrecalcExprs); + *pPrecalcExprs = NULL; + } + return cxt.errCode; +} + +static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode) { + SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG); + CHECK_ALLOC(pAgg, NULL); + + SNodeList* pPrecalcExprs = NULL; + SNodeList* pGroupKeys = NULL; + SNodeList* pAggFuncs = NULL; + CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg); + CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg); + + STupleDescNode* pChildTupe = &(((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple); + // push down expression to outputTuple of child node + if (NULL != pPrecalcExprs) { + pAgg->pExprs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pPrecalcExprs); + CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg); + CHECK_CODE(addTupleDesc(pCxt, pAgg->pExprs, pChildTupe), (SPhysiNode*)pAgg); + } + + if (NULL != pGroupKeys) { + pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->tupleId, -1, pGroupKeys); + CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg); + CHECK_CODE(addTupleDesc(pCxt, pAgg->pGroupKeys, &pAgg->node.outputTuple), (SPhysiNode*)pAgg); + } + + if (NULL != pAggFuncs) { + pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pAggFuncs); + CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg); + CHECK_CODE(addTupleDesc(pCxt, pAgg->pAggFuncs, &pAgg->node.outputTuple), (SPhysiNode*)pAgg); + } + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg); + + CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, &pAgg->node.outputTuple), (SPhysiNode*)pAgg); + + return (SPhysiNode*)pAgg; +} + +static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode) { + SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT); CHECK_ALLOC(pProject, NULL); - SNodeList* pProjections = transformListForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->pProjections); - CHECK_ALLOC(pProjections, (SPhysiNode*)pProject); - CHECK_CODE(addTupleDesc(pCxt, pProjections, &pProject->node.outputTuple, &pProject->pProjections), (SPhysiNode*)pProject); - nodesDestroyList(pProjections); + pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple.tupleId, -1, pProjectLogicNode->pProjections); + CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject); + CHECK_CODE(addTupleDesc(pCxt, pProject->pProjections, &pProject->node.outputTuple), (SPhysiNode*)pProject); - if (NULL != pProjectLogicNode->node.pConditions) { - pProject->node.pConditions = transformForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->node.pConditions); - CHECK_ALLOC(pProject->node.pConditions, (SPhysiNode*)pProject); - } + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject); return (SPhysiNode*)pProject; } static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) { - SNodeList* pChildern = nodesMakeList(); - CHECK_ALLOC(pChildern, NULL); + SNodeList* pChildren = nodesMakeList(); + CHECK_ALLOC(pChildren, NULL); SNode* pLogicChild; FOREACH(pLogicChild, pLogicPlan->pChildren) { SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild); - if (TSDB_CODE_SUCCESS != nodesListAppend(pChildern, pChildPhyNode)) { + if (TSDB_CODE_SUCCESS != nodesListAppend(pChildren, pChildPhyNode)) { pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - nodesDestroyList(pChildern); + nodesDestroyList(pChildren); return NULL; } } @@ -636,22 +818,22 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPl pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan); break; case QUERY_NODE_LOGIC_PLAN_JOIN: + pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan); break; case QUERY_NODE_LOGIC_PLAN_AGG: + pPhyNode = createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicPlan); break; case QUERY_NODE_LOGIC_PLAN_PROJECT: - pPhyNode = createProjectPhysiNode(pCxt, (SProjectLogicNode*)pLogicPlan); + pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan); break; default: break; } - if (NULL != pPhyNode) { - pPhyNode->pChildren = pChildern; - SNode* pChild; - FOREACH(pChild, pPhyNode->pChildren) { - ((SPhysiNode*)pChild)->pParent = pPhyNode; - } + pPhyNode->pChildren = pChildren; + SNode* pChild; + FOREACH(pChild, pPhyNode->pChildren) { + ((SPhysiNode*)pChild)->pParent = pPhyNode; } return pPhyNode; diff --git a/source/libs/planner/test/newPlannerTest.cpp b/source/libs/planner/test/newPlannerTest.cpp index e99f0c150c..51ef52ac2d 100644 --- a/source/libs/planner/test/newPlannerTest.cpp +++ b/source/libs/planner/test/newPlannerTest.cpp @@ -123,8 +123,8 @@ TEST_F(NewPlannerTest, simple) { TEST_F(NewPlannerTest, groupBy) { setDatabase("root", "test"); - bind("SELECT count(*) FROM t1"); - ASSERT_TRUE(run()); + // bind("SELECT count(*) FROM t1"); + // ASSERT_TRUE(run()); bind("SELECT c1, count(*) FROM t1 GROUP BY c1"); ASSERT_TRUE(run()); diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 1dd533c1c5..f51dd66cca 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -307,12 +307,12 @@ typedef struct SFilterInfo { #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) -#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type) -#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes) -#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId) -#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId) -#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc)) -#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnRefNode *)((fi)->desc))->dataType.bytes * (ri)) +#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type) +#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes) +#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId) +#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId) +#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc)) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnNode *)((fi)->desc))->node.resType.bytes * (ri)) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) #define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8f8fc25d18..b50228a3dd 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -886,14 +886,14 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE) { + if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE) { FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } int32_t type; void *v; - if (nodeType(node) == QUERY_NODE_COLUMN_REF) { + if (nodeType(node) == QUERY_NODE_COLUMN) { type = FLD_TYPE_COLUMN; v = node; } else { @@ -1418,7 +1418,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SColumnRefNode *refNode = (SColumnRefNode *)field->desc; + SColumnNode *refNode = (SColumnNode *)field->desc; qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId); } @@ -1447,7 +1447,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) char str[512] = {0}; SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); - SColumnRefNode *refNode = (SColumnRefNode *)left->desc; + SColumnNode *refNode = (SColumnNode *)left->desc; if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_NMATCH){ len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); } @@ -3549,17 +3549,17 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } - if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } } else { - if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { + if ((QUERY_NODE_COLUMN != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } - if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { + if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } @@ -3569,7 +3569,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { SNode *t = node->pLeft; node->pLeft = node->pRight; node->pRight = t; @@ -3582,10 +3582,10 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } if (OP_TYPE_IN != node->opType) { - SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft; + SColumnNode *refNode = (SColumnNode *)node->pLeft; SValueNode *valueNode = (SValueNode *)node->pRight; - int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type); - if (0 != type && type != refNode->dataType.type) { + int32_t type = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type); + if (0 != type && type != refNode->node.resType.type) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index aa29b02709..095a88e040 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -50,13 +50,13 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t //TODO BUILD HASH break; } - case QUERY_NODE_COLUMN_REF: { + case QUERY_NODE_COLUMN: { if (NULL == ctx) { sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SColumnRefNode *ref = (SColumnRefNode *)node; + SColumnNode *ref = (SColumnNode *)node; if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -190,7 +190,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu SScalarFuncExecFuncs ffpSet = {0}; int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); if (code) { - sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError( +"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_RET(code); } @@ -208,7 +209,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu for (int32_t i = 0; i < rowNum; ++i) { code = (*ffpSet.process)(params, node->pParameterList->length, output); if (code) { - sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError( +"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_JRET(code); } From 2fb0af814d3aecffb18f3d7fdfc5c37c9c12e04f Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 24 Feb 2022 05:39:41 -0500 Subject: [PATCH 2/5] merge 3.0 --- source/libs/scalar/src/filter.c | 2 +- source/libs/scalar/src/scalar.c | 2 +- source/libs/scalar/test/filter/filterTests.cpp | 18 +++++++++--------- source/libs/scalar/test/scalar/scalarTests.cpp | 12 ++++++------ 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 92e30f86f2..5eb0003662 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3487,7 +3487,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN_REF == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index e4ad45700c..742c7fd706 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -601,7 +601,7 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) { EDealRes sclCalcWalker(SNode* pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index d27fe2c25f..c8685a71c9 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -76,15 +76,15 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { } void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { - SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); - SColumnRefNode *rnode = (SColumnRefNode *)node; - rnode->dataType.type = dataType; - rnode->dataType.bytes = dataBytes; + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; rnode->tupleId = 0; if (NULL == block) { rnode->slotId = 2; - rnode->columnId = 55; + rnode->colId = 55; *pNode = (SNode *)rnode; return; @@ -120,7 +120,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->columnId = 55; + rnode->colId = 55; *block = res; } else { @@ -137,7 +137,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in res->info.numOfCols++; rnode->slotId = idx; - rnode->columnId = 55 + idx; + rnode->colId = 55 + idx; } *pNode = (SNode *)rnode; @@ -259,7 +259,7 @@ TEST(columnTest, smallint_column_greater_double_value) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - stat.colId = ((SColumnRefNode *)pLeft)->columnId; + stat.colId = ((SColumnNode *)pLeft)->colId; stat.max = 10; stat.min = 5; stat.numOfNull = 0; @@ -310,7 +310,7 @@ TEST(columnTest, int_column_greater_smallint_value) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - stat.colId = ((SColumnRefNode *)pLeft)->columnId; + stat.colId = ((SColumnNode *)pLeft)->colId; stat.max = 10; stat.min = 5; stat.numOfNull = 0; diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index de2c7f5874..5be117065b 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -75,10 +75,10 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { } void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { - SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); - SColumnRefNode *rnode = (SColumnRefNode *)node; - rnode->dataType.type = dataType; - rnode->dataType.bytes = dataBytes; + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; rnode->tupleId = 0; if (NULL == *block) { @@ -111,7 +111,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->columnId = 55; + rnode->colId = 55; *block = res; } else { @@ -126,7 +126,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in taosArrayPush(res->pDataBlock, &idata); rnode->slotId = idx; - rnode->columnId = 55 + idx; + rnode->colId = 55 + idx; } *pNode = (SNode *)rnode; From 930d09dc0b8c34627cf948425a17155d974e4ebe Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 24 Feb 2022 19:19:45 +0800 Subject: [PATCH 3/5] add multi tag query --- source/libs/index/inc/index_util.h | 34 +++--- source/libs/index/src/index.c | 17 +-- source/libs/index/src/index_tfile.c | 2 +- source/libs/index/src/index_util.c | 70 ++++++++++++ source/libs/index/test/CMakeLists.txt | 20 ++++ source/libs/index/test/utilUT.cc | 149 ++++++++++++++++++++++++++ 6 files changed, 273 insertions(+), 19 deletions(-) create mode 100644 source/libs/index/src/index_util.c create mode 100644 source/libs/index/test/utilUT.cc diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index adeb52bb8c..d31ea01c37 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -15,36 +15,46 @@ #ifndef __INDEX_UTIL_H__ #define __INDEX_UTIL_H__ +#include "tarray.h" + #ifdef __cplusplus extern "C" { #endif -#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ - do { \ - memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \ - buf += sizeof(key->mem); \ +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ } while (0) #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ do { \ - memcpy((void*)buf, (void*)key->mem, len); \ + memcpy((void *)buf, (void *)key->mem, len); \ buf += len; \ } while (0) -#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ - do { \ - type c = var; \ - assert(sizeof(type) == sizeof(c)); \ - memcpy((void*)buf, (void*)&c, sizeof(c)); \ - buf += sizeof(c); \ +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(type) == sizeof(c)); \ + memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ } while (0) #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ do { \ - memcpy((void*)buf, (void*)var, len); \ + memcpy((void *)buf, (void *)var, len); \ buf += len; \ } while (0) +/* multi sorted result intersection + * input: [1, 2, 4, 5] + * [2, 3, 4, 5] + * [1, 4, 5] + * output:[4, 5] + */ +void iIntersection(SArray *interResults, SArray *finalResult); + #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 9287a91828..267d57ab61 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -370,22 +370,27 @@ static void indexInterResultsDestroy(SArray* results) { } taosArrayDestroy(results); } + static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { // refactor, merge interResults into fResults by oType - SArray* first = taosArrayGetP(interResults, 0); - taosArraySort(first, uidCompare); - taosArrayRemoveDuplicate(first, uidCompare, NULL); + + for (int i = 0; i < taosArrayGetSize(interResults); i--) { + SArray* t = taosArrayGetP(interResults, i); + taosArraySort(t, uidCompare); + taosArrayRemoveDuplicate(t, uidCompare, NULL); + } if (oType == MUST) { + iIntersection(interResults, fResults); // just one column index, enhance later - taosArrayAddAll(fResults, first); + // taosArrayAddAll(fResults, interResults); } else if (oType == SHOULD) { // just one column index, enhance later - taosArrayAddAll(fResults, first); + taosArrayAddAll(fResults, interResults); // tag1 condistion || tag2 condition } else if (oType == NOT) { // just one column index, enhance later - taosArrayAddAll(fResults, first); + taosArrayAddAll(fResults, interResults); // not use currently } return 0; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index aff976e52b..282059df29 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { return ret; } - IndexTFile* pTfile = (IndexTFile*)tfile; + IndexTFile* pTfile = tfile; SIndexTerm* term = query->term; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c new file mode 100644 index 0000000000..f03d19838e --- /dev/null +++ b/source/libs/index/src/index_util.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "index_util.h" +#include "index.h" +typedef struct MergeIndex { + int idx; + int len; +} MergeIndex; + +static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) { + uint64_t v; + int32_t m; + while (s <= e) { + m = s + (e - s) / 2; + v = *(uint64_t *)taosArrayGet(arr, m); + if (v >= k) { + e = m - 1; + } else { + s = m + 1; + } + } + return s; +} + +void iIntersection(SArray *inters, SArray *final) { + int32_t sz = taosArrayGetSize(inters); + if (sz <= 0) { + return; + } + MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray *t = taosArrayGetP(inters, i); + + mi[i].len = taosArrayGetSize(t); + mi[i].idx = 0; + } + + SArray *base = taosArrayGetP(inters, 0); + for (int i = 0; i < taosArrayGetSize(base); i++) { + uint64_t tgt = *(uint64_t *)taosArrayGet(base, i); + bool has = true; + for (int j = 1; j < taosArrayGetSize(inters); j++) { + SArray *oth = taosArrayGetP(inters, j); + int mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); + if (mid >= 0 && mid < mi[j].len) { + uint64_t val = *(uint64_t *)taosArrayGet(oth, mid); + has = (val == tgt ? true : false); + mi[j].idx = mid; + } else { + has = false; + } + } + if (has == true) { + taosArrayPush(final, &tgt); + } + } + tfree(mi); +} diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 665dfd7318..bed42be3e5 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable(indexTest "") add_executable(fstTest "") add_executable(fstUT "") +add_executable(UtilUT "") target_sources(indexTest PRIVATE @@ -15,6 +16,11 @@ target_sources(fstUT PRIVATE "fstUT.cc" ) +target_sources(UtilUT + PRIVATE + "utilUT.cc" +) + target_include_directories ( indexTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" @@ -31,6 +37,12 @@ target_include_directories ( fstUT "${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + +target_include_directories ( UtilUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries (indexTest os util @@ -53,6 +65,14 @@ target_link_libraries (fstUT index ) +target_link_libraries (UtilUT + os + util + common + gtest_main + index +) + #add_test( # NAME index_test diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc new file mode 100644 index 0000000000..a286965e20 --- /dev/null +++ b/source/libs/index/test/utilUT.cc @@ -0,0 +1,149 @@ +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "index_util.h" +#include "tglobal.h" +#include "tskiplist.h" +#include "tutil.h" + +class UtilEnv : public ::testing::Test { + protected: + virtual void SetUp() { + src = (SArray *)taosArrayInit(2, sizeof(void *)); + for (int i = 0; i < 3; i++) { + SArray *m = taosArrayInit(10, sizeof(uint64_t)); + taosArrayPush(src, &m); + } + + rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t)); + } + virtual void TearDown() { + for (int i = 0; i < taosArrayGetSize(src); i++) { + SArray *m = (SArray *)taosArrayGetP(src, i); + taosArrayDestroy(m); + } + taosArrayDestroy(src); + } + + SArray *src; + SArray *rslt; +}; + +static void clearSourceArray(SArray *p) { + for (int i = 0; i < taosArrayGetSize(p); i++) { + SArray *m = (SArray *)taosArrayGetP(p, i); + taosArrayClear(m); + } +} +static void clearFinalArray(SArray *p) { taosArrayClear(p); } +TEST_F(UtilEnv, intersectionSimpleResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 10); + + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectMultiEmptyResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 10; i < 20; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 20; i < 30; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + // empty source + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectSimpleEmpty) { + clearSourceArray(src); + clearFinalArray(rslt); + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +} +TEST_F(UtilEnv, intersect01) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {2, 3, 4, 5}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {1, 2, 3, 5}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {3, 5, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 2); +} +TEST_F(UtilEnv, intersect02) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {8, 10, 12, 13}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {9, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +} From 9f24aaf4f799e1623ab0d8b9ec8c333c6cee42fd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 24 Feb 2022 22:02:45 +0800 Subject: [PATCH 4/5] add multi tag query --- source/libs/index/inc/index_util.h | 2 +- source/libs/index/src/index.c | 6 +--- source/libs/index/src/index_util.c | 43 +++++++++++++++++++++++- source/libs/index/test/utilUT.cc | 54 ++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 7 deletions(-) diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index d31ea01c37..36830a68bc 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -54,7 +54,7 @@ extern "C" { * output:[4, 5] */ void iIntersection(SArray *interResults, SArray *finalResult); - +void iUnion(SArray *interResults, SArray *finalResult); #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 267d57ab61..5147734a85 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -382,12 +382,8 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType if (oType == MUST) { iIntersection(interResults, fResults); - // just one column index, enhance later - // taosArrayAddAll(fResults, interResults); } else if (oType == SHOULD) { - // just one column index, enhance later - taosArrayAddAll(fResults, interResults); - // tag1 condistion || tag2 condition + iUnion(interResults, fResults); } else if (oType == NOT) { // just one column index, enhance later taosArrayAddAll(fResults, interResults); diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c index f03d19838e..ecf5e6d36c 100644 --- a/source/libs/index/src/index_util.c +++ b/source/libs/index/src/index_util.c @@ -42,7 +42,6 @@ void iIntersection(SArray *inters, SArray *final) { MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); for (int i = 0; i < sz; i++) { SArray *t = taosArrayGetP(inters, i); - mi[i].len = taosArrayGetSize(t); mi[i].idx = 0; } @@ -68,3 +67,45 @@ void iIntersection(SArray *inters, SArray *final) { } tfree(mi); } +void iUnion(SArray *inters, SArray *final) { + int32_t sz = taosArrayGetSize(inters); + if (sz <= 0) { + return; + } + MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray *t = taosArrayGetP(inters, i); + mi[i].len = taosArrayGetSize(t); + mi[i].idx = 0; + } + while (1) { + uint64_t mVal = UINT_MAX; + int mIdx = -1; + + for (int j = 0; j < sz; j++) { + SArray *t = taosArrayGetP(inters, j); + if (mi[j].idx >= mi[j].len) { + continue; + } + uint64_t cVal = *(uint64_t *)taosArrayGet(t, mi[j].idx); + if (cVal < mVal) { + mVal = cVal; + mIdx = j; + } + } + if (mIdx != -1) { + mi[mIdx].idx++; + if (taosArrayGetSize(final) > 0) { + uint64_t lVal = *(uint64_t *)taosArrayGetLast(final); + if (lVal == mVal) { + continue; + } + } + taosArrayPush(final, &mVal); + } else { + break; + } + } + + tfree(mi); +} diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc index a286965e20..8954978344 100644 --- a/source/libs/index/test/utilUT.cc +++ b/source/libs/index/test/utilUT.cc @@ -147,3 +147,57 @@ TEST_F(UtilEnv, intersect02) { iIntersection(src, rslt); assert(taosArrayGetSize(rslt) == 0); } +TEST_F(UtilEnv, 01union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 3); +} +TEST_F(UtilEnv, 02union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {13, 14, 15}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 3); +} +TEST_F(UtilEnv, 03union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 16, 18, 20}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {0, 12, 13, 20, 23}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {1, 12, 13, 16, 17}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 9); +} From 5a37fdf28fb9302f0754effb15a87e88271c07c0 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 25 Feb 2022 01:47:20 +0800 Subject: [PATCH 5/5] [TD-13062]: cross platform file system. --- include/common/ttszip.h | 6 +- include/libs/wal/wal.h | 8 +- include/os/osFile.h | 78 ++- include/util/tfile.h | 26 +- source/common/src/ttszip.c | 135 ++-- source/dnode/mgmt/impl/inc/dndEnv.h | 2 +- source/dnode/mgmt/impl/src/dndBnode.c | 20 +- source/dnode/mgmt/impl/src/dndEnv.c | 28 +- source/dnode/mgmt/impl/src/dndMgmt.c | 21 +- source/dnode/mgmt/impl/src/dndMnode.c | 20 +- source/dnode/mgmt/impl/src/dndQnode.c | 20 +- source/dnode/mgmt/impl/src/dndSnode.c | 20 +- source/dnode/mgmt/impl/src/dndVnodes.c | 20 +- source/dnode/mnode/impl/src/mndTelem.c | 41 +- source/dnode/mnode/impl/test/trans/trans.cpp | 14 +- source/dnode/mnode/sdb/src/sdbFile.c | 50 +- source/dnode/vnode/src/inc/tqInt.h | 4 +- source/dnode/vnode/src/inc/tsdbFile.h | 28 +- source/dnode/vnode/src/meta/metaBDBImpl.c | 1 + source/dnode/vnode/src/tq/tqMetaStore.c | 58 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 45 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 12 +- source/libs/CMakeLists.txt | 1 + source/libs/function/src/taggfunction.c | 8 +- .../index/inc/index_fst_counting_writer.h | 2 +- .../index/src/index_fst_counting_writer.c | 36 +- source/libs/index/src/index_tfile.c | 8 +- source/libs/tdb/src/db/tdbEnv.c | 12 +- source/libs/tdb/src/db/tdbPgFile.c | 12 +- source/libs/tdb/src/inc/tdbPgFile.h | 2 +- source/libs/tfs/test/tfsTest.cpp | 18 +- source/libs/transport/inc/rpcUdp.h | 2 +- source/libs/transport/test/pushServer.c | 16 +- source/libs/transport/test/rserver.c | 16 +- source/libs/wal/src/walMeta.c | 34 +- source/libs/wal/src/walMgmt.c | 14 +- source/libs/wal/src/walRead.c | 44 +- source/libs/wal/src/walSeek.c | 52 +- source/libs/wal/src/walWrite.c | 60 +- source/os/src/osFile.c | 578 ++++++++++-------- source/os/src/osRand.c | 10 +- source/os/src/osSocket.c | 1 + source/os/src/osSysinfo.c | 101 ++- source/os/src/osTimezone.c | 13 +- source/util/src/tconfig.c | 21 +- source/util/src/tfile.c | 210 +++---- source/util/src/tlog.c | 106 ++-- source/util/src/tpagedbuf.c | 27 +- tools/shell/src/backup/shellCheck.c | 8 +- tools/shell/src/backup/shellImport.c | 12 +- tools/shell/src/shellEngine.c | 74 +-- 51 files changed, 1129 insertions(+), 1026 deletions(-) diff --git a/include/common/ttszip.h b/include/common/ttszip.h index 38699ae791..d83b0066d5 100644 --- a/include/common/ttszip.h +++ b/include/common/ttszip.h @@ -73,9 +73,9 @@ typedef struct STSGroupBlockInfoEx { } STSGroupBlockInfoEx; typedef struct STSBuf { - FILE* f; - char path[PATH_MAX]; - uint32_t fileSize; + TdFilePtr pFile; + char path[PATH_MAX]; + uint32_t fileSize; // todo use array STSGroupBlockInfoEx* pData; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 45f1d88c30..f90dbb97fe 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -120,8 +120,8 @@ typedef struct SWal { int32_t fsyncSeq; // meta SWalVer vers; - int64_t writeLogTfd; - int64_t writeIdxTfd; + TdFilePtr pWriteLogTFile; + TdFilePtr pWriteIdxTFile; int32_t writeCur; SArray *fileInfoSet; // status @@ -138,8 +138,8 @@ typedef struct SWal { typedef struct SWalReadHandle { SWal *pWal; - int64_t readLogTfd; - int64_t readIdxTfd; + TdFilePtr pReadLogTFile; + TdFilePtr pReadIdxTFile; int64_t curFileFirstVer; int64_t curVersion; int64_t capacity; diff --git a/include/os/osFile.h b/include/os/osFile.h index ac399fa3be..59492f6694 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -22,49 +22,63 @@ extern "C" { #include "osSocket.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -typedef int32_t FileFd; -#else -typedef int32_t FileFd; +#ifndef ALLOW_FORBID_FUNC + #define open OPEN_FUNC_TAOS_FORBID + #define fopen FOPEN_FUNC_TAOS_FORBID + // #define close CLOSE_FUNC_TAOS_FORBID + // #define fclose FCLOSE_FUNC_TAOS_FORBID #endif -#define FD_INITIALIZER ((int32_t)-1) - #ifndef PATH_MAX #define PATH_MAX 256 #endif -int32_t taosLockFile(FileFd fd); -int32_t taosUnLockFile(FileFd fd); - -int32_t taosUmaskFile(FileFd fd); - +typedef struct TdFile *TdFilePtr; + +#define TD_FILE_CTEATE 0x0001 +#define TD_FILE_WRITE 0x0002 +#define TD_FILE_READ 0x0004 +#define TD_FILE_TRUNC 0x0008 +#define TD_FILE_APPEND 0x0010 +#define TD_FILE_TEXT 0x0020 +#define TD_FILE_AUTO_DEL 0x0040 +#define TD_FILE_EXCL 0x0080 + +int32_t taosLockFile(TdFilePtr pFile); +int32_t taosUnLockFile(TdFilePtr pFile); + +int32_t taosUmaskFile(int32_t maskVal); + int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); -int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime); - -FileFd taosOpenFileWrite(const char *path); -FileFd taosOpenFileCreateWrite(const char *path); -FileFd taosOpenFileCreateWriteTrunc(const char *path); -FileFd taosOpenFileCreateWriteAppend(const char *path); -FileFd taosOpenFileRead(const char *path); -FileFd taosOpenFileReadWrite(const char *path); - -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence); -int32_t taosFtruncateFile(FileFd fd, int64_t length); -int32_t taosFsyncFile(FileFd fd); - -int64_t taosReadFile(FileFd fd, void *buf, int64_t count); -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count); - -void taosCloseFile(FileFd fd); - +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); + +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); + +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length); +int32_t taosFsyncFile(TdFilePtr pFile); + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); +void taosFprintfFile(TdFilePtr pFile, const char *format, ...); +size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); +int32_t taosEOFFile(TdFilePtr pFile); + +int64_t taosCloseFile(TdFilePtr *ppFile); + int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); - + void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); + +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size); +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size); -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); +bool taosValidFile(TdFilePtr pFile); + +int taosGetErrorFile(TdFilePtr pFile); #ifdef __cplusplus } diff --git a/include/util/tfile.h b/include/util/tfile.h index d3813051a4..59953de861 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -30,20 +30,20 @@ void tfCleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused -int64_t tfOpenRead(const char *pathname); -int64_t tfOpenReadWrite(const char *pathname); -int64_t tfOpenCreateWrite(const char *pathname); -int64_t tfOpenCreateWriteAppend(const char *pathname); +// int64_t tfOpenRead(const char *pathname); +// int64_t tfOpenReadWrite(const char *pathname); +// int64_t tfOpenCreateWrite(const char *pathname); +// int64_t tfOpenCreateWriteAppend(const char *pathname); -int64_t tfClose(int64_t tfd); -int64_t tfWrite(int64_t tfd, void *buf, int64_t count); -int64_t tfRead(int64_t tfd, void *buf, int64_t count); -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); -int32_t tfFsync(int64_t tfd); -bool tfValid(int64_t tfd); -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); -int32_t tfFtruncate(int64_t tfd, int64_t length); -void * tfMmapReadOnly(int64_t tfd, int64_t length); +// int64_t tfClose(int64_t tfd); +// int64_t tfWrite(int64_t tfd, void *buf, int64_t count); +// int64_t tfRead(int64_t tfd, void *buf, int64_t count); +// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); +// int32_t tfFsync(int64_t tfd); +// bool tfValid(int64_t tfd); +// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); +// int32_t tfFtruncate(int64_t tfd, int64_t length); +// void * tfMmapReadOnly(int64_t tfd, int64_t length); #ifdef __cplusplus } #endif diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 3265ea5547..f69bfd6e88 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -24,8 +24,9 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { pTSBuf->autoDelete = autoDelete; taosGetTmpfilePath(osTempDir(), "join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "wb+"); - if (pTSBuf->f == NULL) { + // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } @@ -60,8 +61,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - pTSBuf->f = fopen(pTSBuf->path, "rb+"); - if (pTSBuf->f == NULL) { + // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } @@ -72,9 +74,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // validate the file magic number STSBufFileHeader header = {0}; - int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); UNUSED(ret); - size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f); + size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader)); UNUSED(sz); // invalid file @@ -112,8 +114,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - //int64_t pos = ftell(pTSBuf->f); //pos not used - sz = fread(buf, infoSize, 1, pTSBuf->f); + //int64_t pos = ftell(pTSBuf->pFile); //pos not used + sz = taosReadFile(pTSBuf->pFile, buf, infoSize); UNUSED(sz); // the length value for each vnode is not kept in file, so does not set the length value @@ -123,22 +125,22 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { } free(buf); - ret = fseek(pTSBuf->f, 0, SEEK_END); + ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); UNUSED(ret); - struct stat fileStat; - if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { + int64_t file_size; + if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->fileSize = (uint32_t)fileStat.st_size; + pTSBuf->fileSize = (uint32_t)file_size; tsBufResetPos(pTSBuf); // ascending by default pTSBuf->cur.order = TSDB_ORDER_ASC; -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), +// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile), // pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); return pTSBuf; @@ -156,7 +158,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { tfree(pTSBuf->block.payload); if (!pTSBuf->remainOpen) { - fclose(pTSBuf->f); + taosCloseFile(&pTSBuf->pFile); } if (pTSBuf->autoDelete) { @@ -253,7 +255,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); + int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET); assert(r == 0); /* @@ -264,30 +266,30 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * both side has the compressed length is used to support load data forwards/backwords. */ int32_t metaLen = 0; - metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); int32_t trueLen = pBlock->tag.nLen; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); float tfloat = (float)pBlock->tag.d; - metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); } else { trueLen = 0; - metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); } - fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); + taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); - metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); assert(metaLen == getTagAreaLength(&pBlock->tag)); int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; @@ -332,20 +334,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { * the end of each comp data block */ int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); - int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); - size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); - sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR); + size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); UNUSED(sz); pBlock->compLen = pBlock->padding; offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - ret = fseek(pTSBuf->f, -offset, SEEK_CUR); + ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(ret); } - int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); + ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); // NOTE: mix types tags are not supported size_t sz = 0; @@ -356,23 +358,23 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { memset(tp, 0, pBlock->tag.nLen + 1); pBlock->tag.pz = tp; - sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); UNUSED(sz); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { float tfloat = 0; - sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); pBlock->tag.d = (double)tfloat; UNUSED(sz); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value - sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); UNUSED(sz); } - sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); UNUSED(sz); - sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); UNUSED(sz); - sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); if (decomp) { pTSBuf->tsData.len = @@ -381,11 +383,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { } // read the comp length at the length of comp block - sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); assert(pBlock->padding == pBlock->compLen); int32_t n = 0; - sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen)); if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { assert(n == 0); } else { @@ -396,7 +398,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // for backwards traverse, set the start position at the end of previous block if (order == TSDB_ORDER_DESC) { - int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); + int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(r); } @@ -512,7 +514,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num // todo opt performance by cache blocks info static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { return -1; } @@ -531,7 +533,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); + int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR); UNUSED(ret); } @@ -548,7 +550,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo offset = pBlockInfo->offset + pBlockInfo->compLen; } - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } @@ -618,11 +620,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn return -1; } - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } - fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo)); return 0; } @@ -636,19 +638,19 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { } int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { + if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { return -1; } assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); - int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); if (r != 0) { // qError("fseek failed, errno:%d", errno); return -1; } - size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); + size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader)); if (ws != 1) { // qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); return -1; @@ -823,12 +825,12 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { pBlockInfoEx->info.id = id; } - int32_t r = fseek(pDestBuf->f, 0, SEEK_END); + int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END); assert(r == 0); int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); + int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size); if (written == -1 || written != size) { return -1; @@ -839,17 +841,18 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { int32_t oldSize = pDestBuf->fileSize; // file meta data may be cached, close and reopen the file for accurate file size. - fclose(pDestBuf->f); - pDestBuf->f = fopen(pDestBuf->path, "rb+"); - if (pDestBuf->f == NULL) { + taosCloseFile(&pDestBuf->pFile); + // pDestBuf->pFile = fopen(pDestBuf->path, "rb+"); + pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pDestBuf->pFile == NULL) { return -1; } - struct stat fileStat; - if (fstat(fileno(pDestBuf->f), &fileStat) != 0) { + int64_t file_size; + if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) { return -1; } - pDestBuf->fileSize = (uint32_t)fileStat.st_size; + pDestBuf->fileSize = (uint32_t)file_size; assert(pDestBuf->fileSize == oldSize + size); @@ -868,13 +871,13 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); - int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET); if (ret == -1) { // qError("fseek failed, errno:%d", errno); tsBufDestroy(pTSBuf); return NULL; } - size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); + size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len); if (sz != len) { // qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); tsBufDestroy(pTSBuf); @@ -893,7 +896,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ } // TODO taosFsync?? -// if (taosFsync(fileno(pTSBuf->f)) == -1) { +// if (taosFsync(fileno(pTSBuf->pFile)) == -1) { //// qError("fsync failed, errno:%d", errno); // tsBufDestroy(pTSBuf); // return NULL; @@ -1071,15 +1074,15 @@ int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, in *len = 0; *numOfBlocks = 0; - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { + int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); // qError("%p: fseek failed: %s", pSql, tstrerror(code)); return code; } - size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); + size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen); if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); + int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); // tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); return code; } diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 7465dc14f7..cbd5eb5827 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -125,7 +125,7 @@ typedef struct SDnode { EStat stat; SDnodeObjCfg cfg; SDnodeDir dir; - FileFd lockFd; + TdFilePtr pLockFile; SDnodeMgmt dmgmt; SMnodeMgmt mmgmt; SQnodeMgmt qmgmt; diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 610579f6f5..81b020c152 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_BNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_BNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { PRASE_BNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 5bc3b756ea..247f52f958 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); } -static FileFd dndCheckRunning(char *dataDir) { +static TdFilePtr dndCheckRunning(char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); - FileFd fd = taosOpenFileCreateWriteTrunc(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - return -1; + return NULL; } - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - taosCloseFile(fd); - return -1; + taosCloseFile(&pFile); + return NULL; } - return fd; + return pFile; } static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { - pDnode->lockFd = dndCheckRunning(pCfg->dataDir); - if (pDnode->lockFd < 0) { + pDnode->pLockFile = dndCheckRunning(pCfg->dataDir); + if (pDnode->pLockFile == NULL) { return -1; } @@ -147,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) { tfree(pDnode->dir.snode); tfree(pDnode->dir.bnode); - if (pDnode->lockFd >= 0) { - taosUnLockFile(pDnode->lockFd); - taosCloseFile(pDnode->lockFd); - pDnode->lockFd = 0; + if (pDnode->pLockFile != NULL) { + taosUnLockFile(pDnode->pLockFile); + taosCloseFile(&pDnode->pLockFile); + pDnode->pLockFile = NULL; } } diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 4827065e87..e0d7f299b3 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) { int32_t maxLen = 256 * 1024; char *content = calloc(1, maxLen + 1); cJSON *root = NULL; - FILE *fp = NULL; - fp = fopen(pMgmt->file, "r"); - if (fp == NULL) { + // fp = fopen(pMgmt->file, "r"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", pMgmt->file); code = 0; goto PRASE_DNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", pMgmt->file); goto PRASE_DNODE_OVER; @@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) { PRASE_DNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); @@ -309,8 +309,9 @@ PRASE_DNODE_OVER: static int32_t dndWriteDnodes(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; - FILE *fp = fopen(pMgmt->file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(pMgmt->file, "w"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { dError("failed to write %s since %s", pMgmt->file, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index b69516b1b7..e1b16a188f 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_MNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_MNODE_OVER; @@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { PRASE_MNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index e5600fcaf0..93e2209610 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_QNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_QNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { PRASE_QNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index fb108559b1..4906aef246 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_SNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_SNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { PRASE_SNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index a0c0a02431..f20493aa7f 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - fp = fopen(file, "r"); - if (fp == NULL) { + // fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_VNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_VNODE_OVER; @@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ PRASE_VNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); return code; } @@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write %s since %s", file, terrstr()); return -1; @@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 2f7d5e10b6..7c1215ea42 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -62,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { size_t size = 0; int32_t done = 0; - FILE* fp = fopen("/proc/cpuinfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/cpuinfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ); + if (pFile == NULL) { return; } - while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { + while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { const char* v = strchr(line, ':') + 2; @@ -83,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/etc/os-release", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/etc/os-release", "r"); + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "PRETTY_NAME", 11) == 0) { const char* p = strchr(line, '=') + 1; @@ -109,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/proc/meminfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/meminfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "MemTotal", 8) == 0) { const char* p = strchr(line, ':') + 1; @@ -132,8 +135,8 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) { @@ -252,16 +255,16 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { static void mndGetEmail(SMnode* pMnode, char* filepath) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - int32_t fd = taosOpenFileRead(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { return; } - if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { + if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } - taosCloseFile(fd); + taosCloseFile(&pFile); } int32_t mndInitTelem(SMnode* pMnode) { diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 88d4fc4f75..cea93017f4 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test { static void KillThenRestartServer() { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; - FileFd fd = taosOpenFileRead(file); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = malloc(size); - int32_t readLen = taosReadFile(fd, buffer, size); + int32_t readLen = taosReadFile(pFile, buffer, size); if (readLen < 0 || readLen == size) { ASSERT(1); } - taosCloseFile(fd); + taosCloseFile(&pFile); test.ServerStop(); - fd = taosOpenFileCreateWriteTrunc(file); - int32_t writeLen = taosWriteFile(fd, buffer, readLen); + pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + int32_t writeLen = taosWriteFile(pFile, buffer, readLen); if (writeLen < 0 || writeLen == readLen) { ASSERT(1); } free(buffer); - taosFsyncFile(fd); - taosCloseFile(fd); + taosFsyncFile(pFile); + taosCloseFile(&pFile); taosMsleep(1000); test.ServerStart(); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6d17423324..a3fe4dfb14 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) { return 0; } -static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { - int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t)); +static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { + int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t maxId = -1; - ret = taosReadFile(fd, &maxId, sizeof(int64_t)); + ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t ver = -1; - ret = taosReadFile(fd, &ver, sizeof(int64_t)); + ret = taosReadFile(pFile, &ver, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { } char reserve[SDB_RESERVE_SIZE] = {0}; - ret = taosReadFile(fd, reserve, sizeof(reserve)); + ret = taosReadFile(pFile, reserve, sizeof(reserve)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { return 0; } -static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { - if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { +static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { + if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { maxId = pSdb->maxId[i]; } - if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { ver = pSdb->tableVer[i]; } - if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } char reserve[SDB_RESERVE_SIZE] = {0}; - if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) { + if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) { snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); mDebug("start to read file:%s", file); - FileFd fd = taosOpenFileRead(file); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { free(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, terrstr()); return 0; } - if (sdbReadFileHead(pSdb, fd) != 0) { + if (sdbReadFileHead(pSdb, pFile) != 0) { mError("failed to read file:%s head since %s", file, terrstr()); pSdb->curVer = -1; free(pRaw); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } while (1) { readLen = sizeof(SSdbRaw); - ret = taosReadFile(fd, pRaw, readLen); + ret = taosReadFile(pFile, pRaw, readLen); if (ret == 0) break; if (ret < 0) { @@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) { } readLen = pRaw->dataLen + sizeof(int32_t); - ret = taosReadFile(fd, pRaw->pData, readLen); + ret = taosReadFile(pFile, pRaw->pData, readLen); if (ret < 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, tstrerror(code)); @@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) { mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer); PARSE_SDB_DATA_ERROR: - taosCloseFile(fd); + taosCloseFile(&pFile); sdbFreeRaw(pRaw); terrno = code; @@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer, pSdb->lastCommitVer); - FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to open file:%s for write since %s", tmpfile, terrstr()); return -1; } - if (sdbWriteFileHead(pSdb, fd) != 0) { + if (sdbWriteFileHead(pSdb, pFile) != 0) { mError("failed to write file:%s head since %s", tmpfile, terrstr()); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } @@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { if (pRaw != NULL) { pRaw->status = pRow->status; int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen; - if (taosWriteFile(fd, pRaw, writeLen) != writeLen) { + if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); - if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { + if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } if (code == 0) { - code = taosFsyncFile(fd); + code = taosFsyncFile(pFile); if (code != 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to write file:%s since %s", tmpfile, tstrerror(code)); } } - taosCloseFile(fd); + taosCloseFile(&pFile); if (code == 0) { code = taosRenameFile(tmpfile, curfile); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 344ad992f0..e87de10912 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -141,9 +141,9 @@ typedef struct { STqMetaList* unconnectTopic; // TODO:temporaral use, to be replaced by unified tfile - int fileFd; + TdFilePtr pFile; // TODO:temporaral use, to be replaced by unified tfile - int idxFd; + TdFilePtr pIdxFile; char* dirPath; int32_t tqConfigFlag; diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 4a79c6358d..9fe753c7d8 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -28,17 +28,17 @@ #define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_F(tf) (&((tf)->f)) -#define TSDB_FILE_FD(tf) ((tf)->fd) +#define TSDB_FILE_PFILE(tf) ((tf)->pFile) #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) +#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL) #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) -#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) +#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL) #define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level) #define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id) #define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did) #define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname) #define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf)) +#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf)) #define TSDB_FILE_STATE(tf) ((tf)->state) #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) @@ -178,10 +178,10 @@ typedef struct { } SDFInfo; typedef struct { - SDFInfo info; - STfsFile f; - int fd; - uint8_t state; + SDFInfo info; + STfsFile f; + TdFilePtr pFile; + uint8_t state; } SDFile; void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); @@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { ASSERT(!TSDB_FILE_OPENED(pDFile)); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -209,7 +209,7 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { if (TSDB_FILE_OPENED(pDFile)) { - close(pDFile->fd); + taosCloseFile(&pDFile->pFile); TSDB_FILE_SET_CLOSED(pDFile); } } @@ -217,7 +217,7 @@ static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence); + int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); if (loffset < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte); + int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); if (nwrite < nbyte) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nread = taosReadFile(pDFile->fd, buf, nbyte); + int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); if (nread < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index e5ccd02e48..3c464391d2 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #include "db.h" #include "metaDef.h" diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index d40cc2294f..6d5085ee74 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) { } } -static inline int tqSeekLastPage(int fd) { - int offset = lseek(fd, 0, SEEK_END); +static inline int64_t tqSeekLastPage(TdFilePtr pFile) { + int offset = taosLSeekFile(pFile, 0, SEEK_END); int pageNo = offset / TQ_PAGE_SIZE; int curPageOffset = pageNo * TQ_PAGE_SIZE; - return lseek(fd, curPageOffset, SEEK_SET); + return taosLSeekFile(pFile, curPageOffset, SEEK_SET); } // TODO: the struct is tightly coupled with index entry @@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf { char buffer[TQ_IDX_PAGE_BODY_SIZE]; } STqIdxPageBuf; -static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { - int offset = tqSeekLastPage(fd); +static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { + int offset = tqSeekLastPage(pFile); int nBytes; - if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { } ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset); - return lseek(fd, offset, SEEK_SET); + return taosLSeekFile(pFile, offset, SEEK_SET); } STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, @@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F tqError("failed to create dir:%s since %s ", name, terrstr()); } strcat(name, "/" TQ_IDX_NAME); - int idxFd = open(name, O_RDWR | O_CREAT, 0755); - if (idxFd < 0) { + TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pIdxFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s ", name, terrstr()); // free memory return NULL; } - pMeta->idxFd = idxFd; + pMeta->pIdxFile = pIdxFile; pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F strcpy(name, path); strcat(name, "/" TQ_META_NAME); - int fileFd = open(name, O_RDWR | O_CREAT, 0755); - if (fileFd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s", name, terrstr()); return NULL; } - pMeta->fileFd = fileFd; + pMeta->pFile = pFile; pMeta->pSerializer = serializer; pMeta->pDeserializer = deserializer; @@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F int idxRead; int allocated = TQ_PAGE_SIZE; bool readEnd = false; - while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { + while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) { if (idxRead == -1) { // TODO: handle error terrno = TAOS_SYSTEM_ERROR(errno); @@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); - lseek(fileFd, pNode->handle.offset, SEEK_SET); + taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); if (allocated < pNode->handle.serializedSize) { void* ptr = realloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { @@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F allocated = pNode->handle.serializedSize; } serializedObj->ssize = pNode->handle.serializedSize; - if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { + if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { // TODO: read error } if (serializedObj->action == TQ_ACTION_INUSE) { @@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { // commit data and idx tqStorePersist(pMeta); ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { } int32_t tqStoreDelete(STqMetaStore* pMeta) { - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { pSHead->checksum = 0; pSHead->ssize = sizeof(STqSerializedHead); /*int allocatedSize = sizeof(STqSerializedHead);*/ - int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); + int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR); - tqReadLastPage(pMeta->idxFd, &idxBuf); + tqReadLastPage(pMeta->pIdxFile, &idxBuf); if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { - lseek(pMeta->idxFd, 0, SEEK_END); + taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END); memset(&idxBuf, 0, TQ_PAGE_SIZE); idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; } else { @@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); } - nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize); + nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytes == pSHead->ssize); } @@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); } - int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize); + int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytesTxn == pSHead->ssize); nBytes += nBytesTxn; } @@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { idxBuf.head.writeOffset += TQ_IDX_SIZE; if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { - nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); + nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE); // TODO: handle error with tfile ASSERT(nBytes == TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE); @@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { free(pSHead); // TODO: write new version in tfile if ((char*)bufPtr != idxBuf.buffer) { - int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); + int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); // TODO: handle error in tfile ASSERT(nBytes == idxBuf.head.writeOffset); } // TODO: using fsync in tfile - fsync(pMeta->idxFd); - fsync(pMeta->fileFd); + taosFsyncFile(pMeta->pIdxFile); + taosFsyncFile(pMeta->pFile); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index e5d5ef513f..9a444a1c17 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname); - int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE); - if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { // Encode file status and write to file if (fsheader.len > 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbEncodeFSStatus(&ptr, pStatus); taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len); - if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) { + if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); (void)remove(tfname); taosTZfree(pBuf); return -1; @@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { } // fsync, close and rename - if (taosFsyncFile(fd) < 0) { + if (taosFsyncFile(pFile) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); taosTZfree(pBuf); return -1; } - (void)close(fd); + (void)taosCloseFile(&pFile); (void)taosRenameFile(tfname, cfname); taosTZfree(pBuf); @@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { static int tsdbOpenFSFromCurrent(STsdb *pRepo) { STsdbFS * pfs = REPO_FS(pRepo); - int fd = -1; + TdFilePtr pFile = NULL; void * buffer = NULL; SFSHeader fsheader; char current[TSDB_FILENAME_LEN] = "\0"; @@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); // current file exists, try to recover - fd = open(current, O_RDONLY | O_BINARY); - if (fd < 0) { + pFile = taosOpenFile(current, TD_FILE_READ); + if (pFile == NULL) { tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE); + int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE); if (nread < 0) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, strerror(errno)); @@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - nread = (int)taosReadFile(fd, buffer, fsheader.len); + nread = (int)taosReadFile(pFile, buffer, fsheader.len); if (nread < 0) { tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { } taosTZfree(buffer); - close(fd); + taosCloseFile(&pFile); return 0; _err: - if (fd >= 0) { - close(fd); + if (pFile != NULL) { + taosCloseFile(&pFile); } taosTZfree(buffer); return -1; @@ -1244,18 +1244,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } if (tsdbForceKeepFile) { - struct stat tfstat; - + int64_t file_size; // Get real file size - if (fstat(pDFile->fd, &tfstat) < 0) { + if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosArrayDestroy(fArray); return -1; } - if (pDFile->info.size != tfstat.st_size) { + if (pDFile->info.size != file_size) { int64_t tfsize = pDFile->info.size; - pDFile->info.size = tfstat.st_size; + pDFile->info.size = file_size; tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 579160bcb2..ff79d91081 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile < 0) { if (errno == ENOENT) { // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); @@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { } tfree(s); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -460,7 +460,7 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { return -1; } - if (taosFtruncateFile(df.fd, df.info.size) < 0) { + if (taosFtruncateFile(df.pFile, df.info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; @@ -541,7 +541,7 @@ static int tsdbRollBackDFile(SDFile *pDFile) { return -1; } - if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) { + if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index aa6d204310..e07e46948f 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,3 +1,4 @@ +add_definitions("-D ALLOW_FORBID_FUNC") add_subdirectory(transport) add_subdirectory(sync) add_subdirectory(tdb) diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 1aa5871468..2ff2edf84e 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -3960,14 +3960,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) { // qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); // TODO refactor transfer ownership of current file - *(FILE **)pCtx->pOutput = pTSbuf->f; + *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile; pResInfo->complete = true; // get the file size - struct stat fStat; - if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { - pResInfo->numOfRes = fStat.st_size; + int64_t file_size; + if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) { + pResInfo->numOfRes = (uint32_t )file_size; } pTSbuf->remainOpen = true; diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 1e0a88e17f..86e829aa95 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -38,7 +38,7 @@ typedef struct WriterCtx { WriterType type; union { struct { - int fd; + TdFilePtr pFile; bool readOnly; char buf[256]; int size; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index b57f639726..86aa257b48 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -19,7 +19,7 @@ static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { if (ctx->type == TFile) { - assert(len == tfWrite(ctx->file.fd, buf, len)); + assert(len == taosWriteFile(ctx->file.pFile, buf, len)); } else { memcpy(ctx->mem.buf + ctx->offset, buf, len); } @@ -33,7 +33,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { nRead = len < ctx->file.size ? len : ctx->file.size; memcpy(buf, ctx->file.ptr, nRead); #else - nRead = tfRead(ctx->file.fd, buf, len); + nRead = taosReadFile(ctx->file.pFile, buf, len); #endif } else { memcpy(buf, ctx->mem.buf + ctx->offset, len); @@ -45,13 +45,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { int nRead = 0; if (ctx->type == TFile) { - // tfLseek(ctx->file.fd, offset, 0); + // tfLseek(ctx->file.pFile, offset, 0); #ifdef USE_MMAP int32_t last = ctx->file.size - offset; nRead = last >= len ? len : last; memcpy(buf, ctx->file.ptr + offset, nRead); #else - nRead = tfPread(ctx->file.fd, buf, len, offset); + nRead = taosPReadFile(ctx->file.pFile, buf, len, offset); #endif } else { // refactor later @@ -69,9 +69,9 @@ static int writeCtxGetSize(WriterCtx* ctx) { } static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { - // taosFsyncFile(ctx->file.fd); - tfFsync(ctx->file.fd); - // tfFlush(ctx->file.fd); + // taosFsyncFile(ctx->file.pFile); + taosFsyncFile(ctx->file.pFile); + // tfFlush(ctx->file.pFile); } else { // do nothing } @@ -87,25 +87,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ugly code, refactor later ctx->file.readOnly = readOnly; if (readOnly == false) { - // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenCreateWriteAppend(path); - tfFtruncate(ctx->file.fd, 0); - struct stat fstat; - stat(path, &fstat); - ctx->file.size = fstat.st_size; + // ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + taosFtruncateFile(ctx->file.pFile, 0); + int64_t file_size; + taosStatFile(path, &file_size, NULL); + ctx->file.size = (int)file_size; } else { - // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenRead(path); + // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); struct stat fstat; stat(path, &fstat); ctx->file.size = fstat.st_size; #ifdef USE_MMAP - ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size); + ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); #endif } memcpy(ctx->file.buf, path, strlen(path)); - if (ctx->file.fd < 0) { + if (ctx->file.pFile < 0) { indexError("failed to open file, error %d", errno); goto END; } @@ -133,7 +133,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { free(ctx->mem.buf); } else { ctx->flush(ctx); - tfClose(ctx->file.fd); + taosCloseFile(&ctx->file.pFile); if (ctx->file.readOnly) { #ifdef USE_MMAP munmap(ctx->file.ptr, ctx->file.size); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 282059df29..e44f8fc1c3 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -586,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); if (nread == -1) { - indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - errno, reader->ctx->file.fd, reader->ctx->file.buf); + indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), + errno, reader->ctx->file.buf); } else { - indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - reader->ctx->file.fd, reader->ctx->file.buf); + indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), + reader->ctx->file.buf); } // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 3670c770ab..2afce30828 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -18,7 +18,7 @@ struct STDbEnv { char * rootDir; // root directory of the environment char * jname; // journal file name - int jfd; // journal file fd + TdFilePtr jpFile; // journal file fd pgsz_t pgSize; // page size cachesz_t cacheSize; // total cache size STDbList dbList; // TDB List @@ -55,7 +55,7 @@ int tdbEnvCreate(TENV **ppEnv, const char *rootDir) { pEnv->rootDir = (char *)(&pEnv[1]); pEnv->jname = pEnv->rootDir + slen + 1; - pEnv->jfd = -1; + pEnv->jpFile = NULL; pEnv->pgSize = TDB_DEFAULT_PGSIZE; pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE; @@ -139,8 +139,8 @@ static int tdbEnvDestroy(TENV *pEnv) { } int tdbEnvBeginTxn(TENV *pEnv) { - pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755); - if (pEnv->jfd < 0) { + pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pEnv->jpFile < 0) { return -1; } @@ -149,8 +149,8 @@ int tdbEnvBeginTxn(TENV *pEnv) { int tdbEnvCommit(TENV *pEnv) { /* TODO */ - close(pEnv->jfd); - pEnv->jfd = -1; + taosCloseFile(&pEnv->jpFile); + pEnv->jpFile = NULL; return 0; } diff --git a/source/libs/tdb/src/db/tdbPgFile.c b/source/libs/tdb/src/db/tdbPgFile.c index ee5b486f7b..48f8ed1792 100644 --- a/source/libs/tdb/src/db/tdbPgFile.c +++ b/source/libs/tdb/src/db/tdbPgFile.c @@ -51,10 +51,10 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { pPgFile->fname = (char *)(&(pPgFile[1])); memcpy(pPgFile->fname, fname, fnameLen); pPgFile->fname[fnameLen] = '\0'; - pPgFile->fd = -1; + pPgFile->pFile = NULL; - pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755); - if (pPgFile->fd < 0) { + pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pPgFile->pFile == NULL) { // TODO: handle error return -1; } @@ -95,8 +95,8 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { int pgFileClose(SPgFile *pPgFile) { if (pPgFile) { - if (pPgFile->fd >= 0) { - close(pPgFile->fd); + if (pPgFile->pFile >= 0) { + taosCloseFile(&pPgFile->pFile); } tfree(pPgFile->fname); @@ -201,7 +201,7 @@ static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) { pTData = pData; szToRead = pgSize; for (; szToRead > 0;) { - rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize); + rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize); if (rsize < 0) { if (errno == EINTR) { continue; diff --git a/source/libs/tdb/src/inc/tdbPgFile.h b/source/libs/tdb/src/inc/tdbPgFile.h index 2a7116a0dd..eaeebf6b9d 100644 --- a/source/libs/tdb/src/inc/tdbPgFile.h +++ b/source/libs/tdb/src/inc/tdbPgFile.h @@ -38,7 +38,7 @@ struct SPgFile { uint8_t fileid[TDB_FILE_ID_LEN]; // file id pgno_t lsize; // page file logical size (for count) pgno_t fsize; // real file size on disk (for rollback) - int fd; + TdFilePtr pFile; SPgFileListNode envHash; SPgFileListNode envPgfList; }; diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 178d115c59..af66304f84 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -230,10 +230,11 @@ TEST_F(TfsTest, 04_File) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); @@ -638,10 +639,11 @@ TEST_F(TfsTest, 05_MultiDisk) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); diff --git a/source/libs/transport/inc/rpcUdp.h b/source/libs/transport/inc/rpcUdp.h index c1da6a9240..0c651d07ed 100644 --- a/source/libs/transport/inc/rpcUdp.h +++ b/source/libs/transport/inc/rpcUdp.h @@ -30,7 +30,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t void taosFreeMsgHdr(void *hdr); int taosMsgHdrSize(void *hdr); -void taosSendMsgHdr(void *hdr, int fd); +void taosSendMsgHdr(void *hdr, TdFilePtr pFile); void taosInitMsgHdr(void **hdr, void *dest, int maxPkts); void taosSetMsgHdrData(void *hdr, char *data, int dataLen); diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 0ccfde16ce..a1c181ac98 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile != NULL) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -181,8 +181,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CTEATE | TD_FILE_WRITE); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -190,8 +190,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 4337cd242c..794497500c 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile >= 0) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -170,8 +170,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -179,8 +179,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 2d6fb8fc76..ae0b0bd849 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -69,8 +69,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); pLastFileInfo->fileSize = statbuf.st_size; - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -79,15 +79,15 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { char* buf = malloc(readSize + 5); if (buf == NULL) { - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } - taosLSeekFile(fd, -readSize, SEEK_END); - if (readSize != taosReadFile(fd, buf, readSize)) { + taosLSeekFile(pFile, -readSize, SEEK_END); + if (readSize != taosReadFile(pFile, buf, readSize)) { free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -108,12 +108,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { // file has to be deleted free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } } - taosCloseFile(fd); + taosCloseFile(&pFile); SWalHead *lastEntry = (SWalHead*)found; return lastEntry->head.version; @@ -364,18 +364,18 @@ int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer + 1, fnameStr); - FileFd metaFd = taosOpenFileCreateWrite(fnameStr); - if (metaFd < 0) { + TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE); + if (pMataFile == NULL) { return -1; } char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); - if (len != taosWriteFile(metaFd, serialized, len)) { + if (len != taosWriteFile(pMataFile, serialized, len)) { // TODO:clean file return -1; } - taosCloseFile(metaFd); + taosCloseFile(&pMataFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -404,20 +404,20 @@ int walLoadMeta(SWal* pWal) { return -1; } memset(buf, 0, size + 5); - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - if (taosReadFile(fd, buf, size) != size) { + if (taosReadFile(pFile, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return code; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d5c28d9d9b..d3cd23f284 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -89,8 +89,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // open meta walResetVer(&pWal->vers); - pWal->writeLogTfd = -1; - pWal->writeIdxTfd = -1; + pWal->pWriteLogTFile = NULL; + pWal->pWriteIdxTFile = NULL; pWal->writeCur = -1; pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { @@ -164,10 +164,10 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); - tfClose(pWal->writeLogTfd); - pWal->writeLogTfd = -1; - tfClose(pWal->writeIdxTfd); - pWal->writeIdxTfd = -1; + taosCloseFile(&pWal->pWriteLogTFile); + pWal->pWriteLogTFile = NULL; + taosCloseFile(&pWal->pWriteIdxTFile); + pWal->pWriteIdxTFile = NULL; walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; @@ -207,7 +207,7 @@ static void walFsyncAll() { if (walNeedFsync(pWal)) { wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); - int32_t code = tfFsync(pWal->writeLogTfd); + int32_t code = taosFsyncFile(pWal->pWriteLogTFile); if (code != 0) { wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1d9201f69d..159d281759 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -25,8 +25,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } pRead->pWal = pWal; - pRead->readIdxTfd = -1; - pRead->readLogTfd = -1; + pRead->pReadIdxTFile = NULL; + pRead->pReadLogTFile = NULL; pRead->curVersion = -1; pRead->curFileFirstVer = -1; pRead->capacity = 0; @@ -41,8 +41,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } void walCloseReadHandle(SWalReadHandle *pRead) { - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); tfree(pRead->pHead); free(pRead); } @@ -52,24 +52,24 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; - int64_t idxTfd = pRead->readIdxTfd; - int64_t logTfd = pRead->readLogTfd; + TdFilePtr pIdxTFile = pRead->pReadIdxTFile; + TdFilePtr pLogTFile = pRead->pReadLogTFile; // seek position int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); - code = tfLseek(idxTfd, offset, SEEK_SET); + code = taosLSeekFile(pIdxTFile, offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } // TODO:deserialize ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -80,24 +80,24 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { char fnameStr[WAL_FILE_LEN]; - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); - int64_t logTfd = tfOpenRead(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); - int64_t idxTfd = tfOpenRead(fnameStr); - if (idxTfd < 0) { + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pIdxTFile == NULL) { return -1; } - pRead->readLogTfd = logTfd; - pRead->readIdxTfd = idxTfd; + pRead->pReadLogTFile = pLogTFile; + pRead->pReadIdxTFile = pIdxTFile; return 0; } @@ -145,9 +145,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } - if (!tfValid(pRead->readLogTfd)) return -1; + if (!taosValidFile(pRead->pReadLogTFile)) return -1; - code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); + code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { return -1; } @@ -165,7 +165,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { pRead->pHead = ptr; pRead->capacity = pRead->pHead->head.len; } - if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { + if (pRead->pHead->head.len != taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.len)) { return -1; } @@ -202,7 +202,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { } *ppHead = ptr; } - if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { return -1; } // TODO: endian compatibility processing after read @@ -215,7 +215,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { *ppHead = NULL; return -1; } - if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { + if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } // TODO: endian compatibility processing after read diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 91b172444b..7aae9138c1 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -23,25 +23,25 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd = pWal->writeIdxTfd; - int64_t logTfd = pWal->writeLogTfd; + TdFilePtr pIdxTFile = pWal->pWriteIdxTFile; + TdFilePtr pLogTFile = pWal->pWriteLogTFile; // seek position int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; // TODO:deserialize - code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)); + code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,43 +50,43 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { } int walSetWrite(SWal* pWal) { - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; return 0; } int walChangeWrite(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -103,23 +103,23 @@ int walChangeWrite(SWal* pWal, int64_t ver) { int64_t fileFirstVer = pFileInfo->firstVer; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeIdxTfd = -1; + pWal->pWriteIdxTFile = NULL; return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { - tfClose(idxTfd); + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile < 0) { + taosCloseFile(&pIdxTFile); terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeLogTfd = -1; + pWal->pWriteLogTFile = NULL; return -1; } - pWal->writeLogTfd = logTfd; - pWal->writeIdxTfd = idxTfd; + pWal->pWriteLogTFile = pLogTFile; + pWal->pWriteIdxTFile = pIdxTFile; pWal->writeCur = idx; return fileFirstVer; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a4b34dee37..f9bb168234 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -65,15 +65,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t idxTfd = tfOpenReadWrite(fnameStr); + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); // TODO:change to deserialize function - if (idxTfd < 0) { + if (pIdxTFile == NULL) { pthread_mutex_unlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; @@ -81,20 +81,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // read idx file and get log file pos // TODO:change to deserialize function SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { pthread_mutex_unlock(&pWal->mutex); return -1; } ASSERT(entry.ver == ver); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t logTfd = tfOpenReadWrite(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); + if (pLogTFile < 0) { // TODO pthread_mutex_unlock(&pWal->mutex); return -1; } - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { // TODO pthread_mutex_unlock(&pWal->mutex); @@ -102,8 +102,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } // validate offset SWalHead head; - ASSERT(tfValid(logTfd)); - int size = tfRead(logTfd, &head, sizeof(SWalHead)); + ASSERT(taosValidFile(pLogTFile)); + int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead)); if (size != sizeof(SWalHead)) { return -1; } @@ -118,11 +118,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return -1; } // truncate old files - code = tfFtruncate(logTfd, entry.offset); + code = taosFtruncateFile(pLogTFile, entry.offset); if (code < 0) { return -1; } - code = tfFtruncate(idxTfd, idxOff); + code = taosFtruncateFile(pIdxTFile, idxOff); if (code < 0) { return -1; } @@ -203,31 +203,31 @@ int32_t walEndSnapshot(SWal *pWal) { int walRoll(SWal *pWal) { int code = 0; - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { return -1; } } - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { return -1; } } - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; // create new file int64_t newFileFirstVersion = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, newFileFirstVersion, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -237,8 +237,8 @@ int walRoll(SWal *pWal) { } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; ASSERT(pWal->writeCur >= 0); @@ -248,7 +248,7 @@ int walRoll(SWal *pWal) { static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry)); + int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); // TODO truncate @@ -282,16 +282,16 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in // must truncate explicitly first return -1; } - /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ + /*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/ ASSERT(pWal->writeCur >= 0); pthread_mutex_lock(&pWal->mutex); - if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) { + if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) { walSetWrite(pWal); - tfLseek(pWal->writeLogTfd, 0, SEEK_END); - tfLseek(pWal->writeIdxTfd, 0, SEEK_END); + taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END); + taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END); } pWal->writeHead.head.version = index; @@ -302,14 +302,14 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) { + if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), @@ -336,7 +336,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); - if (tfFsync(pWal->writeLogTfd) < 0) { + if (taosFsyncFile(pWal->pWriteLogTFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5a67ab73eb..4bfe618f2c 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -12,38 +12,45 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -#include + #include -#if defined(_MSDOS) -#define open _open -#endif + #if defined(_MSDOS) + #define open _open + #endif -#if defined(_WIN32) -extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */ -extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ -#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) -#define open openU -#else /* _ANSI_SOURCE */ -#define open openA -#endif /* defined(_UTF8_SOURCE) */ -#endif /* defined(_WIN32) */ + #if defined(_WIN32) + extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */ + extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ + #if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) + #define open openU + #else /* _ANSI_SOURCE */ + #define open openA + #endif /* defined(_UTF8_SOURCE) */ + #endif /* defined(_WIN32) */ #else -#include -#include -#include -#include -#include + #include + #include + #include + #include + #include + #define LINUX_FILE_NO_TEXT_OPTION 0 + #define O_TEXT LINUX_FILE_NO_TEXT_OPTION #endif -void taosCloseFile(FileFd fd) { - close(fd); - fd = FD_INITIALIZER; -} +typedef int32_t FileFd; + +typedef struct TdFile { + int refId; + FileFd fd; + FILE *fp; +}*TdFilePtr,TdFile; + + void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -96,13 +103,166 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch #endif } -int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { +int64_t taosCopyFile(const char *from, const char *to) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + char buffer[4096]; + int64_t size = 0; + int64_t bytes; + + // fidfrom = open(from, O_RDONLY); + TdFilePtr pFileFrom = taosOpenFile(from,TD_FILE_READ); + if (pFileFrom == NULL) goto _err; + + // fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); + TdFilePtr pFileTo = taosOpenFile(to,TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_EXCL); + if (pFileTo == NULL) goto _err; + + while (true) { + bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer)); + if (bytes < 0) goto _err; + if (bytes == 0) break; + + size += bytes; + + if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err; + if (bytes < sizeof(buffer)) break; + } + + taosFsyncFile(pFileTo); + + taosCloseFile(&pFileFrom); + taosCloseFile(&pFileTo); + return size; + +_err: + if (pFileFrom != NULL) taosCloseFile(&pFileFrom); + if (pFileTo != NULL) taosCloseFile(&pFileTo); + remove(to); + return -1; +#endif +} + +int32_t taosRenameFile(const char *oldName, const char *newName) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); + if (code < 0) { + //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#else + int32_t code = rename(oldName, newName); + if (code < 0) { + //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#endif +} + +int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); + if (code < 0) { + return code; + } + + if (size != NULL) { + *size = fileStat.st_size; + } + + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } + + return 0; +#endif +} + +void autoDelFileListAdd(const char *path) { + return; +} + +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return NULL; +#else + int access = O_BINARY; + char *mode = NULL; + access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0; + if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) + { + access |= O_RDWR; + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; + }else if(tdFileOptions & TD_FILE_WRITE) { + access |= O_WRONLY; + mode = (tdFileOptions & TD_FILE_TEXT) ? "wt" : "wb"; + }else if(tdFileOptions & TD_FILE_READ) { + access |= O_RDONLY; + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb"; + } + access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0; + access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; + access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; + access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + if(tdFileOptions & TD_FILE_AUTO_DEL) { + autoDelFileListAdd(path); + } + int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); + if(fd == -1) { + return NULL; + } + FILE* fp = fdopen(fd, mode); + if (fp == NULL) { + close(fd); + return NULL; + } + TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile)); + if (pFile == NULL) { + close(fd); + fclose(fp); + return NULL; + } + pFile->fd = fd; + pFile->fp = fp; + pFile->refId = 0; + return pFile; +#endif +} + +int64_t taosCloseFile(TdFilePtr *ppFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + if(ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { + return 0; + } + fsync((*ppFile)->fd); + close((*ppFile)->fd); + (*ppFile)->fd = -1; + (*ppFile)->fp = NULL; + (*ppFile)->refId = 0; + free(*ppFile); + *ppFile = NULL; + return 0; +#endif +} + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { + if(pFile == NULL) { + return 0; + } int64_t leftbytes = count; int64_t readbytes; char * tbuf = (char *)buf; while (leftbytes > 0) { - readbytes = read(fd, (void *)tbuf, (uint32_t)leftbytes); + readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); if (readbytes < 0) { if (errno == EINTR) { continue; @@ -120,13 +280,20 @@ int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { return count; } -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { - int64_t nleft = n; +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { + if(pFile == NULL) { + return 0; + } + return pread(pFile->fd, buf, count, offset); +} + +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { + int64_t nleft = count; int64_t nwritten = 0; char * tbuf = (char *)buf; while (nleft > 0) { - nwritten = write(fd, (void *)tbuf, (uint32_t)nleft); + nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft); if (nwritten < 0) { if (errno == EINTR) { continue; @@ -137,48 +304,119 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { tbuf += nwritten; } - return n; + return count; } -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { + return (int64_t)lseek(pFile->fd, (long)offset, whence); +} -int64_t taosCopyFile(const char *from, const char *to) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - char buffer[4096]; - int fidto = -1, fidfrom = -1; - int64_t size = 0; - int64_t bytes; - - fidfrom = open(from, O_RDONLY); - if (fidfrom < 0) goto _err; - - fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); - if (fidto < 0) goto _err; - - while (true) { - bytes = taosReadFile(fidfrom, buffer, sizeof(buffer)); - if (bytes < 0) goto _err; - if (bytes == 0) break; - - size += bytes; - - if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err; - if (bytes < sizeof(buffer)) break; + struct stat fileStat; + int32_t code = fstat(pFile->fd, &fileStat); + if (code < 0) { + return code; } - taosFsyncFile(fidto); + if (size != NULL) { + *size = fileStat.st_size; + } - taosCloseFile(fidfrom); - taosCloseFile(fidto); - return size; + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } -_err: - if (fidfrom >= 0) taosCloseFile(fidfrom); - if (fidto >= 0) taosCloseFile(fidto); - remove(to); - return -1; + return 0; +#endif +} + +int32_t taosLockFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); +#endif +} + +int32_t taosUnLockFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); +#endif +} + +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); + if (!cur) { + uError("SetFilePointerEx Error getting current position in file.\n"); + return -1; + } + + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + uError("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + if (!SetEndOfFile(h)) { + int error = GetLastError(); + uError("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + return 0; +#else + return ftruncate(pFile->fd, l_size); +#endif +} + +int32_t taosFsyncFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + return FlushFileBuffers(h); +#else + return fsync(pFile->fd); #endif } @@ -301,12 +539,12 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { #else -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; while (leftbytes > 0) { - sentbytes = sendfile(dfd, sfd, offset, leftbytes); + sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes); if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { continue; @@ -323,215 +561,49 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { return size; } -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) { - return taosSendFile(fileno(outfile), fileno(infile), offset, size); +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + return taosSendFile(pFileOut->fd, pFileIn, offset, size); } #endif -int32_t taosFtruncateFile(FileFd fd, int64_t l_size) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; - } - HANDLE h = (HANDLE)_get_osfhandle(fd); - - LARGE_INTEGER li_0; - li_0.QuadPart = (int64_t)0; - BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); - if (!cur) { - uError("SetFilePointerEx Error getting current position in file.\n"); - return -1; - } - - LARGE_INTEGER li_size; - li_size.QuadPart = l_size; - BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); - if (cur2 == 0) { - int error = GetLastError(); - uError("SetFilePointerEx GetLastError is: %d\n", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - if (!SetEndOfFile(h)) { - int error = GetLastError(); - uError("SetEndOfFile GetLastError is:%d", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - return 0; -#else - return ftruncate(fd, l_size); +#ifdef __GNUC__ + __attribute__((format(printf, 2, 3))) #endif +void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { + va_list ap; + va_start(ap, format); + fprintf(pFile->fp, format, ap); + va_end(ap); + fflush(pFile->fp); } -int32_t taosFsyncFile(FileFd fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; - } +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { + if (pFile == NULL) return NULL; - HANDLE h = (HANDLE)_get_osfhandle(fd); - - return FlushFileBuffers(h); -#else - return fsync(fd); -#endif + void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); + return ptr; } -int32_t taosRenameFile(const char *oldName, const char *newName) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } - - return code; -#else - int32_t code = rename(oldName, newName); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } - - return code; -#endif +bool taosValidFile(TdFilePtr pFile) { + return pFile != NULL; } -int32_t taosLockFile(int32_t fd) { +int32_t taosUmaskFile(int32_t maskVal) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - return (int32_t)flock(fd, LOCK_EX | LOCK_NB); + return umask(maskVal); #endif } -int32_t taosUnLockFile(int32_t fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return (int32_t)flock(fd, LOCK_UN | LOCK_NB); -#endif +int taosGetErrorFile(TdFilePtr pFile) { + return errno; } - -int32_t taosUmaskFile(int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return umask(val); -#endif +size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf) { + return getline(ptrBuf, NULL, pFile->fp); } - -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = stat(path, &fileStat); - if (code < 0) { - return code; - } - - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; - } - - return 0; -#endif -} - -int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = fstat(fd, &fileStat); - if (code < 0) { - return code; - } - - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; - } - - return 0; -#endif -} - -int32_t taosOpenFileWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteTrunc(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteAppend(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileRead(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileReadWrite(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - - +int32_t taosEOFFile(TdFilePtr pFile) { + return feof(pFile->fp); +} \ No newline at end of file diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 973323a346..b81e41b3cf 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -24,18 +24,18 @@ uint32_t taosRand(void) { return rand(); } uint32_t taosSafeRand(void) { - int fd; + TdFilePtr pFile; int seed; - fd = open("/dev/urandom", 0); - if (fd < 0) { + pFile = taosOpenFile("/dev/urandom", TD_FILE_READ); + if (pFile == NULL) { seed = (int)time(0); } else { - int len = read(fd, &seed, sizeof(seed)); + int len = taosReadFile(pFile, &seed, sizeof(seed)); if (len < 0) { seed = (int)time(0); } - close(fd); + taosCloseFile(&pFile); } return (uint32_t)seed; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 8afb4ae8b7..07d30276b7 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 5bbd7e59a5..ed289c6934 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -408,19 +408,17 @@ bool taosGetSysMemory(float *memoryUsedMB) { } bool taosGetProcMemory(float *memoryUsedMB) { - FILE *fp = fopen(tsProcMemFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcMemFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcMemFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -431,7 +429,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { if (line == NULL) { //printf("read file:%s failed", tsProcMemFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -440,24 +438,24 @@ bool taosGetProcMemory(float *memoryUsedMB) { sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { - FILE *fp = fopen(tsSysCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsSysCpuFile); return false; } - size_t len; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsSysCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -465,24 +463,24 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { - FILE *fp = fopen(tsProcCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcCpuFile); return false; } - size_t len = 0; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsProcCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -495,8 +493,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } @@ -556,19 +554,17 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes = 0; - FILE *fp = fopen(tsSysNetFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysNetFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsSysNetFile); return false; } ssize_t _bytes = 0; - size_t len = 2048; - char * line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); + char * line = NULL; + while (!taosEOFFile(pFile)) { int64_t o_rbytes = 0; int64_t rpackts = 0; int64_t o_tbytes = 0; @@ -581,12 +577,12 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { int64_t nouse6 = 0; char nouse0[200] = {0}; - _bytes = getline(&line, &len, fp); + _bytes = taosGetLineFile(pFile, &line); if (_bytes < 0) { break; } - line[len - 1] = 0; + line[_bytes - 1] = 0; if (strstr(line, "lo:") != NULL) { continue; @@ -601,8 +597,8 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes += (o_rbytes + o_tbytes); } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } @@ -644,22 +640,20 @@ bool taosGetBandSpeed(float *bandSpeedKb) { } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { - FILE *fp = fopen(tsProcIOFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcIOFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcIOFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; char tmp[10]; int readIndex = 0; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -675,8 +669,8 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { if (readIndex >= 2) break; } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); if (readIndex < 2) { //printf("read file:%s failed", tsProcIOFile); @@ -839,15 +833,15 @@ void taosSetCoreDump(bool enable) { } int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { - int fd; int len = 0; - fd = open("/proc/sys/kernel/random/uuid", 0); - if (fd < 0) { + // fd = open("/proc/sys/kernel/random/uuid", 0); + TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ); + if (pFile == NULL) { return -1; } else { - len = read(fd, uid, uidlen); - close(fd); + len = taosReadFile(pFile, uid, uidlen); + taosCloseFile(&pFile); } if (len >= 36) { @@ -862,16 +856,17 @@ char *taosGetCmdlineByPID(int pid) { static char cmdline[1024]; sprintf(cmdline, "/proc/%d/cmdline", pid); - int fd = open(cmdline, O_RDONLY); - if (fd >= 0) { - int n = read(fd, cmdline, sizeof(cmdline) - 1); + // int fd = open(cmdline, O_RDONLY); + TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ); + if (pFile != NULL) { + int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1); if (n < 0) n = 0; if (n > 0 && cmdline[n - 1] == '\n') --n; cmdline[n] = 0; - close(fd); + taosCloseFile(&pFile); } else { cmdline[0] = 0; } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index a0ea01596f..07846781ad 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -150,17 +150,18 @@ void taosGetSystemTimezone(char *outTimezone) { localtime_r(&tx1, &tm1); /* load time zone string from /etc/timezone */ - FILE *f = fopen("/etc/timezone", "r"); + // FILE *f = fopen("/etc/timezone", "r"); + TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); char buf[68] = {0}; - if (f != NULL) { - int len = fread(buf, 64, 1, f); - if (len < 64 && ferror(f)) { - fclose(f); + if (pFile != NULL) { + int len = taosReadFile(pFile, buf, 64); + if (len < 64 && taosGetErrorFile(pFile)) { + taosCloseFile(&pFile); // printf("read /etc/timezone error, reason:%s", strerror(errno)); return; } - fclose(f); + taosCloseFile(&pFile); buf[sizeof(buf) - 1] = 0; char *lineEnd = strstr(buf, "\n"); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 0f4ba419f3..7ad08e44d5 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -571,31 +571,28 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { } int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { - char *line, *name, *value, *value2, *value3; + char *line = NULL, *name, *value, *value2, *value3; int olen, vlen, vlen2, vlen3; ssize_t _bytes = 0; - size_t len = 1024; - FILE *fp = fopen(filepath, "r"); - if (fp == NULL) { + // FILE *fp = fopen(filepath, "r"); + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - line = malloc(len); - - while (!feof(fp)) { - memset(line, 0, len); + while (!taosEOFFile(pFile)) { name = value = value2 = value3 = NULL; olen = vlen = vlen2 = vlen3 = 0; - _bytes = tgetline(&line, &len, fp); + _bytes = taosGetLineFile(pFile, &line); if (_bytes < 0) { break; } - line[len - 1] = 0; + line[_bytes - 1] = 0; paGetToken(line, &name, &olen); if (olen == 0) continue; @@ -616,8 +613,8 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { // taosReadConfigOption(name, value, value2, value3); } - fclose(fp); - tfree(line); + taosCloseFile(&pFile); + if(line != NULL) tfree(line); uInfo("load from cfg file %s success", filepath); return 0; diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 0f68e9204d..d10ea6a934 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -24,147 +24,147 @@ static int32_t tsFileRsetId = -1; static int8_t tfInited = 0; -static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); } +// static void tfCloseFile(void *p) { taosCloseFile((TdFilePtr)(uintptr_t)p); } int32_t tfInit() { - int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); - if (old == 1) return 0; - tsFileRsetId = taosOpenRef(2000, tfCloseFile); - if (tsFileRsetId > 0) { - return 0; - } else { - atomic_store_8(&tfInited, 0); - return -1; - } + // int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); + // if (old == 1) return 0; + // tsFileRsetId = taosOpenRef(2000, tfCloseFile); + // if (tsFileRsetId > 0) { + // return 0; + // } else { + // atomic_store_8(&tfInited, 0); + // return -1; + // } } void tfCleanup() { - atomic_store_8(&tfInited, 0); - if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); - tsFileRsetId = -1; + // atomic_store_8(&tfInited, 0); + // if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); + // tsFileRsetId = -1; } -static int64_t tfOpenImp(int32_t fd) { - if (fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } +// static int64_t tfOpenImp(TdFilePtr pFile) { +// if (pFile == NULL) { +// terrno = TAOS_SYSTEM_ERROR(errno); +// return -1; +// } - void * p = (void *)(int64_t)fd; - int64_t rid = taosAddRef(tsFileRsetId, p); - if (rid < 0) taosCloseFile(fd); +// void * p = (void *)(int64_t)pFile; +// int64_t rid = taosAddRef(tsFileRsetId, p); +// if (rid < 0) taosCloseFile(&pFile); - return rid; -} +// return rid; +// } -int64_t tfOpenRead(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileRead(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenRead(const char *pathname, int32_t flags) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ); +// return tfOpenImp(fd); +// } -int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileReadWrite(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ | TD_FILE_WRITE); +// return tfOpenImp(fd); +// } -int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWrite(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE); +// return tfOpenImp(fd); +// } -int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWriteAppend(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); +// return tfOpenImp(fd); +// } -int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } +// int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } -int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = taosWriteFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = taosWriteFile(pFile, buf, count); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int64_t tfRead(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfRead(int64_t tfd, void *buf, int64_t count) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = taosReadFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = taosReadFile(pFile, buf, count); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = pread(fd, buf, count, offset); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = pread(fd, buf, count, offset); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int32_t tfFsync(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int32_t tfFsync(int64_t tfd) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFsyncFile(fd); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int32_t code = taosFsyncFile(pFile); - taosReleaseRef(tsFileRsetId, tfd); - return code; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return code; +// } -bool tfValid(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return false; +// bool tfValid(int64_t tfd) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return false; - taosReleaseRef(tsFileRsetId, tfd); - return true; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return true; +// } -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int64_t ret = taosLSeekFile(fd, offset, whence); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int64_t ret = taosLSeekFile(fd, offset, whence); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int32_t tfFtruncate(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int32_t tfFtruncate(int64_t tfd, int64_t length) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFtruncateFile(fd, length); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int32_t code = taosFtruncateFile(fd, length); - taosReleaseRef(tsFileRsetId, tfd); - return code; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return code; +// } -void *tfMmapReadOnly(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return NULL; - int32_t fd = (int32_t)(uintptr_t)p; +// void *tfMmapReadOnly(int64_t tfd, int64_t length) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return NULL; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); - taosReleaseRef(tsFileRsetId, tfd); - return ptr; -} +// void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); +// taosReleaseRef(tsFileRsetId, tfd); +// return ptr; +// } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index ed55ef91b0..a69cf215d9 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -47,7 +47,7 @@ typedef struct { int32_t buffEnd; int32_t buffSize; int32_t minBuffSize; - int32_t fd; + TdFilePtr pFile; int32_t stop; pthread_t asyncThread; pthread_mutex_t buffMutex; @@ -101,7 +101,7 @@ static SLogObj tsLogObj = {.fileNum = 1}; static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); -static void taosCloseLogByFd(int32_t oldFd); +static void taosCloseLogByFd(TdFilePtr pFile); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); extern void taosPrintCfg(); static int32_t taosCompressFile(char *srcFileName, char *destFileName); @@ -150,11 +150,11 @@ void taosCloseLog() { // taosCloseLog(); } -static bool taosLockLogFile(int32_t fd) { - if (fd < 0) return false; +static bool taosLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return false; if (tsLogObj.fileNum > 1) { - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret == 0) { return true; } @@ -163,11 +163,11 @@ static bool taosLockLogFile(int32_t fd) { return false; } -static void taosUnLockLogFile(int32_t fd) { - if (fd < 0) return; +static void taosUnLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return; if (tsLogObj.fileNum > 1) { - taosUnLockFile(fd); + taosUnLockFile(pFile); } } @@ -201,22 +201,22 @@ static void *taosThreadToOpenNewFile(void *param) { taosUmaskFile(0); - int32_t fd = taosOpenFileCreateWriteTrunc(name); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { tsLogObj.openInProgress = 0; tsLogObj.lines = tsLogObj.maxLines - 1000; - uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno)); + uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); return NULL; } - taosLockLogFile(fd); - (void)taosLSeekFile(fd, 0, SEEK_SET); + taosLockLogFile(pFile); + (void)taosLSeekFile(pFile, 0, SEEK_SET); - int32_t oldFd = tsLogObj.logHandle->fd; - tsLogObj.logHandle->fd = fd; + TdFilePtr pOldFile = tsLogObj.logHandle->pFile; + tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - taosCloseLogByFd(oldFd); + taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); @@ -262,8 +262,8 @@ void taosResetLog() { } static bool taosCheckFileIsOpen(char *logFileName) { - int32_t fd = taosOpenFileWrite(logFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE); + if (pFile == NULL) { if (errno == ENOENT) { return false; } else { @@ -272,12 +272,12 @@ static bool taosCheckFileIsOpen(char *logFileName) { } } - if (taosLockLogFile(fd)) { - taosUnLockLogFile(fd); - taosCloseFile(fd); + if (taosLockLogFile(pFile)) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); return false; } else { - taosCloseFile(fd); + taosCloseFile(&pFile); return true; } } @@ -350,31 +350,31 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { pthread_mutex_init(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - tsLogObj.logHandle->fd = taosOpenFileCreateWrite(fileName); + tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (tsLogObj.logHandle->fd < 0) { + if (tsLogObj.logHandle->pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - taosLockLogFile(tsLogObj.logHandle->fd); + taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines int64_t filesize = 0; - if (taosFStatFile(tsLogObj.logHandle->fd, &filesize, NULL) < 0) { + if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) { printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } size = (int32_t)filesize; tsLogObj.lines = size / 60; - taosLSeekFile(tsLogObj.logHandle->fd, 0, SEEK_END); + taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, " new log file \n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); return 0; } @@ -416,11 +416,11 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -430,7 +430,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } void taosDumpData(unsigned char *msg, int32_t len) { @@ -445,7 +445,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { pos += 3; if (c >= 16) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); c = 0; pos = 0; } @@ -453,7 +453,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { @@ -483,11 +483,11 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -497,19 +497,19 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } #if 0 void taosCloseLog() { - taosCloseLogByFd(tsLogObj.logHandle->fd); + taosCloseLogByFd(tsLogObj.logHandle->pFile); } #endif -static void taosCloseLogByFd(int32_t fd) { - if (fd >= 0) { - taosUnLockLogFile(fd); - taosCloseFile(fd); +static void taosCloseLogByFd(TdFilePtr pFile) { + if (pFile != NULL) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); } } @@ -645,12 +645,12 @@ static void taosWriteLog(SLogBuff *tLogBuff) { } if (start < end) { - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff), end); } dbgWN++; @@ -707,17 +707,17 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t ret = 0; int32_t len = 0; char *data = malloc(compressSize); - FILE *srcFp = NULL; // gzFile dstFp = NULL; - srcFp = fopen(srcFileName, "r"); - if (srcFp == NULL) { + // srcFp = fopen(srcFileName, "r"); + TdFilePtr pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ); + if (pSrcFile == NULL) { ret = -1; goto cmp_end; } - int32_t fd = taosOpenFileCreateWriteTrunc(destFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(destFileName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { ret = -2; goto cmp_end; } @@ -735,8 +735,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { // } cmp_end: - if (srcFp) { - fclose(srcFp); + if (pSrcFile) { + taosCloseFile(&pSrcFile); } // if (dstFp) { // gzclose(dstFp); diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 0e8d85492c..ca260f0df5 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -32,7 +32,7 @@ typedef struct SDiskbasedBuf { int32_t numOfPages; int64_t totalBufSize; uint64_t fileSize; // disk file size - FILE* file; + TdFilePtr pFile; int32_t allocateId; // allocated page id char* path; // file path int32_t pageSize; // current used page size @@ -67,7 +67,7 @@ static void printStatisData(const SDiskbasedBuf* pBuf); pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. pResBuf->allocateId = -1; pResBuf->comp = true; - pResBuf->file = NULL; + pResBuf->pFile = NULL; pResBuf->qId = qId; pResBuf->fileSize = 0; @@ -94,8 +94,9 @@ static void printStatisData(const SDiskbasedBuf* pBuf); } static int32_t createDiskFile(SDiskbasedBuf* pBuf) { - pBuf->file = fopen(pBuf->path, "wb+"); - if (pBuf->file == NULL) { + // pBuf->file = fopen(pBuf->path, "wb+"); + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pBuf->pFile == NULL) { // qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } @@ -168,13 +169,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { pg->offset = allocatePositionInFile(pBuf, size); pBuf->nextPos += size; - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - ret = (int32_t) fwrite(t, 1, size, pBuf->file); + ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); if (ret != size) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -199,13 +200,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { } // 3. write to disk. - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - ret = (int32_t)fwrite(t, 1, size, pBuf->file); + ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); if (ret != size) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -233,7 +234,7 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { int32_t ret = TSDB_CODE_SUCCESS; assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); - if (pBuf->file == NULL) { + if (pBuf->pFile == NULL) { if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) { terrno = ret; return NULL; @@ -245,14 +246,14 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { // load file block data in disk static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { ret = TAOS_SYSTEM_ERROR(errno); return ret; } SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); - ret = (int32_t)fread(pPage->data, 1, pg->length, pBuf->file); + ret = (int32_t)taosReadFile(pBuf->pFile, pPage->data, pg->length); if (ret != pg->length) { ret = TAOS_SYSTEM_ERROR(errno); return ret; @@ -495,12 +496,12 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) { printStatisData(pBuf); - if (pBuf->file != NULL) { + if (pBuf->pFile != NULL) { uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n", pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId); - fclose(pBuf->file); + taosCloseFile(&pBuf->pFile); } else { uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId); } diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 7fc8b1409a..33d25b6746 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -116,7 +116,7 @@ static void *shellCheckThreadFp(void *arg) { char file[32] = {0}; snprintf(file, 32, "tb%d.txt", pThread->threadIndex); - FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (!fp) { fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno)); return NULL; @@ -133,7 +133,7 @@ static void *shellCheckThreadFp(void *arg) { int32_t code = taos_errno(pSql); if (code != 0) { int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname); - fwrite(sql, 1, len, fp); + taosWriteFile(pFile, sql, len); atomic_add_fetch_32(&errorNum, 1); } @@ -145,8 +145,8 @@ static void *shellCheckThreadFp(void *arg) { taos_free_result(pSql); } - taosFsync(fileno(fp)); - fclose(fp); + taosFsync(pFile); + taosCloseFile(&pFile); return NULL; } diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index 222d69e854..ce15212f86 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -138,7 +138,6 @@ static void shellSourceFile(TAOS *con, char *fptr) { char * cmd = malloc(tsMaxSQLStringLen); size_t cmd_len = 0; char * line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -171,8 +170,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); @@ -182,7 +182,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stdout, "begin import file:%s\n", fname); int lineNo = 0; - while ((read_len = getline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { ++lineNo; if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; @@ -215,9 +215,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void* shellImportThreadFp(void *arg) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 33774bdd05..906a9ce5f0 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -459,44 +459,44 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { return buf; } -static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) { +static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) { if (val == NULL) { - fprintf(fp, "%s", TSDB_DATA_NULL_STR); + taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR); return; } char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: - fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); + taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); break; case TSDB_DATA_TYPE_TINYINT: - fprintf(fp, "%d", *((int8_t *)val)); + taosFprintfFile(pFile, "%d", *((int8_t *)val)); break; case TSDB_DATA_TYPE_SMALLINT: - fprintf(fp, "%d", *((int16_t *)val)); + taosFprintfFile(pFile, "%d", *((int16_t *)val)); break; case TSDB_DATA_TYPE_INT: - fprintf(fp, "%d", *((int32_t *)val)); + taosFprintfFile(pFile, "%d", *((int32_t *)val)); break; case TSDB_DATA_TYPE_BIGINT: - fprintf(fp, "%" PRId64, *((int64_t *)val)); + taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - fprintf(fp, "%.5f", GET_FLOAT_VAL(val)); + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - fprintf(fp, "%.9f", GET_DOUBLE_VAL(val)); + taosFprintfFile(pFile, "%.9f", GET_DOUBLE_VAL(val)); break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memcpy(buf, val, length); buf[length] = 0; - fprintf(fp, "\'%s\'", buf); + taosFprintfFile(pFile, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: formatTimestamp(buf, *(int64_t *)val, precision); - fprintf(fp, "'%s'", buf); + taosFprintfFile(pFile, "'%s'", buf); break; default: break; @@ -516,8 +516,9 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { return -1; } - FILE *fp = fopen(full_path.we_wordv[0], "w"); - if (fp == NULL) { + // FILE *fp = fopen(full_path.we_wordv[0], "w"); + TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); wordfree(&full_path); return -1; @@ -531,29 +532,29 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { for (int col = 0; col < num_fields; col++) { if (col > 0) { - fprintf(fp, ","); + taosFprintfFile(pFile, ","); } - fprintf(fp, "%s", fields[col].name); + taosFprintfFile(pFile, "%s", fields[col].name); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); int numOfRows = 0; do { int32_t *length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { if (i > 0) { - fputc(',', fp); + taosFprintfFile(pFile, "\n"); } - dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision); + dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); numOfRows++; row = taos_fetch_row(tres); } while (row != NULL); result = 0; - fclose(fp); + taosCloseFile(&pFile); return numOfRows; } @@ -897,14 +898,14 @@ void read_history() { history.hstart = 0; history.hend = 0; char *line = NULL; - size_t line_size = 0; int read_size = 0; char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "r"); - if (f == NULL) { + // FILE *f = fopen(f_history, "r"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ); + if (pFile == NULL) { #ifndef WINDOWS if (errno != ENOENT) { fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); @@ -913,7 +914,7 @@ void read_history() { return; } - while ((read_size = tgetline(&line, &line_size, f)) != -1) { + while ((read_size = taosGetLineFile(pFile, &line)) != -1) { line[read_size - 1] = '\0'; history.hist[history.hend] = strdup(line); @@ -924,16 +925,17 @@ void read_history() { } } - free(line); - fclose(f); + if(line != NULL) free(line); + taosCloseFile(&pFile); } void write_history() { char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "w"); - if (f == NULL) { + // FILE *f = fopen(f_history, "w"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { #ifndef WINDOWS fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); #endif @@ -942,12 +944,12 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { - fprintf(f, "%s\n", history.hist[i]); + taosFprintfFile(pFile, "%s\n", history.hist[i]); tfree(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } - fclose(f); + taosCloseFile(&pFile); } void taos_error(TAOS_RES *tres, int64_t st) { @@ -969,7 +971,6 @@ void source_file(TAOS *con, char *fptr) { char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -989,15 +990,16 @@ void source_file(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); return; } - while ((read_len = tgetline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; @@ -1020,9 +1022,9 @@ void source_file(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void shellGetGrantInfo(void *con) {