diff --git a/2.0/src/client/src/tscGlobalmerge.c b/2.0/src/client/src/tscGlobalmerge.c index 6acbfe3e89..c8ff36db51 100644 --- a/2.0/src/client/src/tscGlobalmerge.c +++ b/2.0/src/client/src/tscGlobalmerge.c @@ -701,7 +701,7 @@ SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint // todo remove it int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { - int32_t maxRowSize = MAX(rowSize, finalRowSize); + int32_t maxRowSize = TMAX(rowSize, finalRowSize); char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize)); size_t size = tscNumOfFields(pQueryInfo); @@ -965,7 +965,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) { w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pRes->info.rows - 1)); if (pOperator->pRuntimeEnv->pQueryAttr->order.order == TSDB_ORDER_DESC) { - SWAP(w->skey, w->ekey, TSKEY); + TSWAP(w->skey, w->ekey, TSKEY); assert(w->skey <= w->ekey); } } diff --git a/2.0/src/client/src/tscParseLineProtocol.c b/2.0/src/client/src/tscParseLineProtocol.c index 20d52fbd3e..2225b2c580 100644 --- a/2.0/src/client/src/tscParseLineProtocol.c +++ b/2.0/src/client/src/tscParseLineProtocol.c @@ -47,7 +47,7 @@ int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; int kvLen1 = (int)strlen(kv1->key); int kvLen2 = (int)strlen(kv2->key); - int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + int res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); if (res != 0) { return res; } else { @@ -123,7 +123,7 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra if (code != 0) { return code; } - pField->bytes = MAX(pField->bytes, bytes); + pField->bytes = TMAX(pField->bytes, bytes); } else { SSchema field = {0}; diff --git a/2.0/src/client/src/tscSQLParser.c b/2.0/src/client/src/tscSQLParser.c index c68d7cef06..1c50a2823e 100644 --- a/2.0/src/client/src/tscSQLParser.c +++ b/2.0/src/client/src/tscSQLParser.c @@ -1785,7 +1785,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32 getNewResColId(pCmd), sizeof(double), false); char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->exprToken.z; - size_t len = MIN(sizeof(pExpr->base.aliasName), pItem->pNode->exprToken.n + 1); + size_t len = TMIN(sizeof(pExpr->base.aliasName), pItem->pNode->exprToken.n + 1); tstrncpy(pExpr->base.aliasName, name, len); tExprNode* pNode = NULL; @@ -2246,7 +2246,7 @@ SSchema tGetUserSpecifiedColumnSchema(SVariant* pVal, SStrToken* exprStr, const if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - size_t tlen = MIN(sizeof(s.name), exprStr->n + 1); + size_t tlen = TMIN(sizeof(s.name), exprStr->n + 1); tstrncpy(s.name, exprStr->z, tlen); strdequote(s.name); } @@ -2419,7 +2419,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); } else if (multiCols) { char uname[TSDB_COL_NAME_LEN] = {0}; - int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); + int32_t len = TMIN(pToken->n + 1, TSDB_COL_NAME_LEN); tstrncpy(uname, pToken->z, len); if (tsKeepOriginalColumnName) { // keep the original column name @@ -2432,7 +2432,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT tstrncpy(name, tmp, TSDB_COL_NAME_LEN); } } else { // use the user-input result column name - int32_t len = MIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); + int32_t len = TMIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); tstrncpy(name, pItem->pNode->exprToken.z, len); } } @@ -4380,7 +4380,7 @@ static void exchangeExpr(tSqlExpr* pExpr) { } pExpr->tokenId = optr; - SWAP(pExpr->pLeft, pExpr->pRight, void*); + TSWAP(pExpr->pLeft, pExpr->pRight, void*); } } @@ -8089,7 +8089,7 @@ int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } - int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); + int64_t timeRange = TABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); int64_t intervalRange = 0; if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { diff --git a/2.0/src/client/src/tscSql.c b/2.0/src/client/src/tscSql.c index ab1fffd5a2..e84bc393a5 100644 --- a/2.0/src/client/src/tscSql.c +++ b/2.0/src/client/src/tscSql.c @@ -51,7 +51,7 @@ static bool validPassword(const char* passwd) { } static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, - uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { + void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { if (taos_init()) { return NULL; } @@ -120,7 +120,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa pObj->signature = pObj; pObj->pRpcObj = (SRpcObj *)pRpcObj; tstrncpy(pObj->user, user, sizeof(pObj->user)); - secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); + secretEncryptLen = TMIN(secretEncryptLen, sizeof(pObj->pass)); memcpy(pObj->pass, secretEncrypt, secretEncryptLen); if (db) { @@ -186,7 +186,7 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) { TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, port, syncConnCallback, NULL, (void **)&pObj); + SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, syncConnCallback, NULL, (void **)&pObj); if (pSql != NULL) { pSql->fp = syncConnCallback; pSql->param = pSql; @@ -245,10 +245,10 @@ TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t us char userBuf[TSDB_USER_LEN] = {0}; char passBuf[TSDB_KEY_LEN] = {0}; char dbBuf[TSDB_DB_NAME_LEN] = {0}; - strncpy(ipBuf, ip, MIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userBuf, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passBuf, pass, MIN(TSDB_KEY_LEN - 1, passLen)); - strncpy(dbBuf, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + strncpy(ipBuf, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userBuf, user, TMIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passBuf, pass, TMIN(TSDB_KEY_LEN - 1, passLen)); + strncpy(dbBuf, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); } @@ -262,7 +262,7 @@ static void asyncConnCallback(void *param, TAOS_RES *tres, int code) { TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, (void **)&pObj); + SSqlObj *pSql = taosConnectImpl(ip, user, pass, NULL, db, asyncConnCallback, param, (void **)&pObj); if (pSql == NULL) { return NULL; } diff --git a/2.0/src/client/src/tscSubquery.c b/2.0/src/client/src/tscSubquery.c index 24a6377a73..8ed7eb08e6 100644 --- a/2.0/src/client/src/tscSubquery.c +++ b/2.0/src/client/src/tscSubquery.c @@ -3396,7 +3396,7 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { } int32_t remain = (int32_t)(pSub->res.numOfRows - pSub->res.row); - numOfRes = (int32_t)(MIN(numOfRes, remain)); + numOfRes = (int32_t)(TMIN(numOfRes, remain)); } if (numOfRes == 0) { // no result any more, free all subquery objects diff --git a/2.0/src/query/src/qAggMain.c b/2.0/src/query/src/qAggMain.c index d57532ac8f..7ab3855506 100644 --- a/2.0/src/query/src/qAggMain.c +++ b/2.0/src/query/src/qAggMain.c @@ -373,7 +373,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *interBytes = (int16_t)sizeof(SPercentileInfo); } else if (functionId == TSDB_FUNC_LEASTSQR) { *type = TSDB_DATA_TYPE_BINARY; - *bytes = MAX(TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string + *bytes = TMAX(TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string *interBytes = *bytes; } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { *type = TSDB_DATA_TYPE_BINARY; @@ -1275,7 +1275,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { continue; \ } \ (num) += 1; \ - (r) += POW2(((type *)d)[i] - (delta)); \ + (r) += TPOW2(((type *)d)[i] - (delta)); \ } static void stddev_function(SQLFunctionCtx *pCtx) { @@ -1314,7 +1314,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { continue; } num += 1; - *retVal += POW2(((int32_t *)pData)[i] - avg); + *retVal += TPOW2(((int32_t *)pData)[i] - avg); } break; } @@ -1427,7 +1427,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { continue; } num += 1; - *retVal += POW2(((int32_t *)pData)[i] - avg); + *retVal += TPOW2(((int32_t *)pData)[i] - avg); } break; } @@ -4109,8 +4109,8 @@ static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockD pDist->totalRows += pSrc->totalRows; if (pResInfo->hasResult == DATA_SET_FLAG) { - pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows); - pDist->minRows = MIN(pDist->minRows, pSrc->minRows); + pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows); + pDist->minRows = TMIN(pDist->minRows, pSrc->minRows); } else { pDist->maxRows = pSrc->maxRows; pDist->minRows = pSrc->minRows; diff --git a/2.0/src/query/src/qExecutor.c b/2.0/src/query/src/qExecutor.c index a6a9115b2f..fdadf39d4d 100644 --- a/2.0/src/query/src/qExecutor.c +++ b/2.0/src/query/src/qExecutor.c @@ -328,7 +328,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO idata.info.bytes = pExpr[i].base.resBytes; idata.info.colId = pExpr[i].base.resColId; - int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform taosArrayPush(res->pDataBlock, &idata); } @@ -2643,7 +2643,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool pQueryAttr->order.order = TSDB_ORDER_ASC; if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->needReverseScan = false; @@ -2653,7 +2653,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { pQueryAttr->order.order = TSDB_ORDER_ASC; if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->needReverseScan = false; @@ -2664,7 +2664,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->order.order = TSDB_ORDER_ASC; @@ -2677,7 +2677,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2688,7 +2688,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2703,7 +2703,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2714,7 +2714,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2754,8 +2754,8 @@ static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; - TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); if (QUERY_IS_ASC_QUERY(pQueryAttr)) { getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); @@ -3523,7 +3523,7 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) return; } - SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; SWITCH_ORDER(pTableQueryInfo->cur.order); @@ -3730,7 +3730,7 @@ static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo } // reverse order time range - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); SET_REVERSE_SCAN_FLAG(pRuntimeEnv); setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); @@ -4109,8 +4109,8 @@ void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { */ STimeWindow w = TSWINDOW_INITIALIZER; - TSKEY sk = MIN(win.skey, win.ekey); - TSKEY ek = MAX(win.skey, win.ekey); + TSKEY sk = TMIN(win.skey, win.ekey); + TSKEY ek = TMAX(win.skey, win.ekey); getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4235,7 +4235,7 @@ static void updateNumOfRowsInResultRows(SQueryRuntimeEnv* pRuntimeEnv, SQLFuncti } SResultRowCellInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); + pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); } } } @@ -6945,8 +6945,8 @@ SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorIn SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); STimeWindow w = TSWINDOW_INITIALIZER; - TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); pInfo->pFillInfo = diff --git a/cmake/traft_CMakeLists.txt.in b/cmake/traft_CMakeLists.txt.in index 9b571b3666..1dd1323603 100644 --- a/cmake/traft_CMakeLists.txt.in +++ b/cmake/traft_CMakeLists.txt.in @@ -7,7 +7,7 @@ ExternalProject_Add(traft BINARY_DIR "${CMAKE_CONTRIB_DIR}/traft" #BUILD_IN_SOURCE TRUE # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/ - CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example + CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure BUILD_COMMAND "$(MAKE)" INSTALL_COMMAND "" TEST_COMMAND "" diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 0c71113056..eacaeb9524 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -20,7 +20,7 @@ if(${BUILD_WITH_CRAFT}) endif(${BUILD_WITH_CRAFT}) if(${BUILD_WITH_TRAFT}) - add_subdirectory(traft) + # add_subdirectory(traft) endif(${BUILD_WITH_TRAFT}) add_subdirectory(tdev) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index cc30cd78f5..58ede62b23 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -136,7 +136,7 @@ typedef uint64_t TKEY; #define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) #define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) #define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) -#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) +#define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) #define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) #define MIN_TS_KEY ((TSKEY)0x8000000000000001) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a80a77f0c4..b3d595d73b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -100,7 +100,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_CLUSTER, TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_TP, - TSDB_MGMT_TABLE_FUNCTION, + TSDB_MGMT_TABLE_FUNC, TSDB_MGMT_TABLE_MAX, } EShowType; @@ -526,20 +526,21 @@ typedef struct { typedef struct { char name[TSDB_FUNC_NAME_LEN]; + int8_t igExists; int8_t funcType; int8_t scriptType; - int8_t align; int8_t outputType; int32_t outputLen; int32_t bufSize; - int64_t sigature; + int64_t signature; int32_t commentSize; int32_t codeSize; char pCont[]; } SCreateFuncReq; typedef struct { - char name[TSDB_FUNC_NAME_LEN]; + char name[TSDB_FUNC_NAME_LEN]; + int8_t igNotExists; } SDropFuncReq; typedef struct { @@ -549,13 +550,13 @@ typedef struct { typedef struct { char name[TSDB_FUNC_NAME_LEN]; + int8_t align; int8_t funcType; int8_t scriptType; - int8_t align; int8_t outputType; int32_t outputLen; int32_t bufSize; - int64_t sigature; + int64_t signature; int32_t commentSize; int32_t codeSize; char pCont[]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 2ff4d58e0b..26d8bc81eb 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -119,9 +119,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DB, "mnode-alter-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SYNC_DB, "mnode-sync-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_DB, "mnode-compact-db", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNCTION, "mnode-create-function", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNCTION, "mnode-retrieve-function", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNCTION, "mnode-drop-function", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNC, "mnode-create-func", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNC, "mnode-retrieve-func", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNC, "mnode-drop-func", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index ecfa344f85..1d193888c9 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -74,6 +74,7 @@ void columnListCopy(SArray* dst, const SArray* src, uint64_t uid); void columnListDestroy(SArray* pColumnList); void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel); +void dropOneLevelExprInfo(SArray* pExprInfo); typedef struct SSourceParam { SArray *pExprNodeList; //Array diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 3262b9437c..5a0da46490 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -72,15 +72,15 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. * This is a asynchronized API, and is also thread-safety. - * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr + * @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob); +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob); /** * Fetch query result from the remote query executor @@ -88,7 +88,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, * @param data * @return */ -int32_t scheduleFetchRows(struct SSchJob *pJob, void **data); +int32_t schedulerFetchRows(struct SSchJob *pJob, void **data); /** @@ -102,7 +102,7 @@ int32_t scheduleFetchRows(struct SSchJob *pJob, void **data); * Free the query job * @param pJob */ -void scheduleFreeJob(void *pJob); +void schedulerFreeJob(void *pJob); void schedulerDestroy(void); diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index e1f28092cf..289cfd9b04 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -33,7 +33,7 @@ typedef struct { typedef struct { SDiskID did; - char aname[TSDB_FILENAME_LEN]; // ABS name + char aname[TSDB_FILENAME_LEN]; // TABS name char rname[TSDB_FILENAME_LEN]; // REL name STfs *pTfs; } STfsFile; diff --git a/include/os/osMath.h b/include/os/osMath.h index 440d3bc787..948fbbf665 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -20,37 +20,37 @@ extern "C" { #endif -#define POW2(x) ((x) * (x)) -#define ABS(x) ((x) > 0 ? (x) : -(x)) +#define TPOW2(x) ((x) * (x)) +#define TABS(x) ((x) > 0 ? (x) : -(x)) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #define SWAP(a, b, c) \ + #define TSWAP(a, b, c) \ do { \ c __tmp = (c)(a); \ (a) = (c)(b); \ (b) = __tmp; \ } while (0) - #define MAX(a, b) (((a) > (b)) ? (a) : (b)) - #define MIN(a, b) (((a) < (b)) ? (a) : (b)) + #define TMAX(a, b) (((a) > (b)) ? (a) : (b)) + #define TMIN(a, b) (((a) < (b)) ? (a) : (b)) #else - #define SWAP(a, b, c) \ + #define TSWAP(a, b, c) \ do { \ typeof(a) __tmp = (a); \ (a) = (b); \ (b) = __tmp; \ } while (0) - #define MAX(a, b) \ + #define TMAX(a, b) \ ({ \ typeof(a) __a = (a); \ typeof(b) __b = (b); \ (__a > __b) ? __a : __b; \ }) - #define MIN(a, b) \ + #define TMIN(a, b) \ ({ \ typeof(a) __a = (a); \ typeof(b) __b = (b); \ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 570c1d8375..69bf085491 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -239,6 +239,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x03C4) #define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x03C5) #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x03C6) +#define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x03C7) // mnode-trans #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) @@ -354,7 +355,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0710) //"Scheduler not exist") #define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0711) //"Task not exist") #define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0712) //"Task already exist") -#define TSDB_CODE_QRY_RES_CACHE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) //"Task result cache not exist") +#define TSDB_CODE_QRY_TASK_CTX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) //"Task context not exist") #define TSDB_CODE_QRY_TASK_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x0714) //"Task cancelled") #define TSDB_CODE_QRY_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0715) //"Task dropped") #define TSDB_CODE_QRY_TASK_CANCELLING TAOS_DEF_ERROR_CODE(0, 0x0716) //"Task cancelling") diff --git a/include/util/tdef.h b/include/util/tdef.h index 4c29d9963d..4d3ef06bd6 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -173,6 +173,7 @@ do { \ #define TSDB_FUNC_BUF_SIZE 512 #define TSDB_FUNC_TYPE_SCALAR 1 #define TSDB_FUNC_TYPE_AGGREGATE 2 +#define TSDB_FUNC_MAX_RETRIEVE 1024 #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) diff --git a/include/util/tlist.h b/include/util/tlist.h index 06d0abc797..134873a993 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -34,6 +34,7 @@ extern "C" { #define TD_SLIST_HEAD(sl) ((sl)->sl_head_) #define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) +#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) ((sln)->field.sl_next_) #define TD_SLIST_INIT(sl) \ do { \ @@ -48,12 +49,25 @@ extern "C" { TD_SLIST_NELES(sl) += 1; \ } while (0) +#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, field) \ + do { \ + TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + TD_SLIST_NELES(sl) += 1; \ + } while (0) + #define TD_SLIST_POP(sl) \ do { \ TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ TD_SLIST_NELES(sl) -= 1; \ } while (0) +#define TD_SLIST_POP_WITH_FIELD(sl, field) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), field); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) + // Double linked list ================ #define TD_DLIST_NODE(TYPE) \ struct { \ @@ -70,6 +84,8 @@ extern "C" { #define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) #define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) ((dln)->field.dl_prev_) +#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) ((dln)->field.dl_next_) #define TD_DLIST_HEAD(dl) ((dl)->dl_head_) #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) #define TD_DLIST_NELES(dl) ((dl)->dl_neles_) @@ -94,6 +110,20 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) +#define TD_DLIST_APPEND_WITH_FIELD(dl, dln, field) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), field) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + #define TD_DLIST_PREPEND(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ @@ -108,6 +138,20 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) +#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, field) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), field) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + #define TD_DLIST_POP(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == (dln)) { \ @@ -126,6 +170,26 @@ extern "C" { TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ } while (0) +#define TD_DLIST_POP_WITH_FIELD(dl, dln, field) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \ + } \ + if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) != NULL) { \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, field), field) = \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \ + } \ + if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) != NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field), field) = \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + } while (0) + // General double linked list typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T; diff --git a/packaging/release.sh b/packaging/release.sh new file mode 100755 index 0000000000..8ea30223a6 --- /dev/null +++ b/packaging/release.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os + +set -e +#set -x + +# set parameters by default value +version="3.0.0.0" + +curr_dir=$(pwd) + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/..)" + +echo "=======================new version number: ${verNumber}======================================" + +build_time=$(date +"%F %R") + +echo "top_dir: ${top_dir}" + +cd ${top_dir} +git pull || : + +echo "curr_dir: ${curr_dir}" + +# 2. cmake executable file +compile_dir="${top_dir}/debug" +if [ -d ${compile_dir} ]; then + rm -rf ${compile_dir} +fi + +mkdir -p ${compile_dir} + +cd ${compile_dir} + +echo "compile_dir: ${compile_dir}" + +cmake .. +make -j32 + +release_dir="${top_dir}/release" +if [ -d ${release_dir} ]; then + rm -rf ${release_dir} +fi + +mkdir -p ${release_dir} +cd ${release_dir} + +install_dir="${release_dir}/TDengine-server-${version}" +mkdir -p ${install_dir} +mkdir -p ${install_dir}/lib + +bin_files="${compile_dir}/source/dnode/mgmt/daemon/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table" +cp ${bin_files} ${install_dir}/ && chmod a+x ${install_dir}/* || : + + +cp ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/snode/libsnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/bnode/libbnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/wal/libwal.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/scheduler/libscheduler.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/planner/libplanner.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/parser/libparser.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/qcom/libqcom.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/transport/libtransport.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/function/libfunction.so ${install_dir}/lib/ +cp ${compile_dir}/source/common/libcommon.so ${install_dir}/lib/ +cp ${compile_dir}/source/os/libos.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/mnode/sdb/libsdb.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/catalog/libcatalog.so ${install_dir}/lib/ + +pkg_name=${install_dir}-Linux-x64 + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : + + diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 296fbba634..c61f3da6bd 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -101,13 +101,13 @@ struct SAppInstInfo { }; typedef struct SAppInfo { - int64_t startTime; - char appName[TSDB_APP_NAME_LEN]; - char *ep; - int32_t pid; - int32_t numOfThreads; - - SHashObj *pInstMap; + int64_t startTime; + char appName[TSDB_APP_NAME_LEN]; + char *ep; + int32_t pid; + int32_t numOfThreads; + SHashObj *pInstMap; + pthread_mutex_t mutex; } SAppInfo; typedef struct STscObj { @@ -192,7 +192,7 @@ uint64_t generateRequestId(); void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -char *getConnectionDB(STscObj* pObj); +char *getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); void taos_init_imp(void); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d2696fb355..28faa76785 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -67,7 +67,9 @@ static void deregisterRequest(SRequestObj* pRequest) { int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); - tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); + int64_t duration = taosGetTimestampMs() - pRequest->metric.start; + tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id, + pRequest->requestId, duration, num, currentInst); taosReleaseRef(clientConnRefPool, pTscObj->id); } @@ -110,7 +112,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.user = (char *)user; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.ckey = "key"; -// rpcInit.spi = 1; + rpcInit.spi = 1; rpcInit.secret = (char *)auth; void* pDnodeConn = rpcOpen(&rpcInit); @@ -251,10 +253,11 @@ void taos_init_imp(void) { clientReqRefPool = taosOpenRef(40960, doDestroyRequest); taosGetAppName(appInfo.appName, NULL); + pthread_mutex_init(&appInfo.mutex, NULL); + appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - tscDebug("client is initialized successfully"); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f10275ff97..471b9b68cb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -58,7 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i return strdup(key); } -static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); +static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); static void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { @@ -110,9 +110,11 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } char* key = getClusterKey(user, secretEncrypt, ip, port); + SAppInstInfo** pInst = NULL; - // TODO: race condition here. - SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + pthread_mutex_lock(&appInfo.mutex); + + pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL) { SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; @@ -123,8 +125,10 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, pInst = &p; } + pthread_mutex_unlock(&appInfo.mutex); + tfree(key); - return taosConnectImpl(user, &secretEncrypt[0], localDb, port, NULL, NULL, *pInst); + return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst); } int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest) { @@ -155,7 +159,7 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { SParseContext cxt = { .requestId = pRequest->requestId, .acctId = pTscObj->acctId, - .db = getConnectionDB(pTscObj), + .db = getDbOfConnection(pTscObj), .pSql = pRequest->sqlstr, .sqlLen = pRequest->sqlLen, .pMsg = pRequest->msgBuf, @@ -214,7 +218,10 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag, if (pQueryNode->type == TSDB_SQL_SELECT) { setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols); + tfree(pSchema); pRequest->type = TDMT_VND_QUERY; + } else { + tfree(pSchema); } return code; @@ -236,13 +243,12 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - - int32_t code = scheduleExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); + int32_t code = schedulerExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry } else { if (pRequest->body.pQueryJob != NULL) { - scheduleFreeJob(pRequest->body.pQueryJob); + schedulerFreeJob(pRequest->body.pQueryJob); } } @@ -251,7 +257,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) return pRequest->code; } - return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob); + return schedulerAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob); } @@ -665,12 +671,14 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQueryNode)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return); } else { + CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); pRequest->code = terrno; } _return: + taosArrayDestroy(pNodeList); qDestroyQuery(pQueryNode); if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { pRequest->code = terrno; @@ -715,7 +723,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe return 0; } -STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) { +STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) { STscObj *pTscObj = createTscObj(user, auth, db, pAppInfo); if (NULL == pTscObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -736,7 +744,7 @@ STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uin tsem_wait(&pRequest->body.rspSem); if (pRequest->code != TSDB_CODE_SUCCESS) { - const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); + const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code); printf("failed to connect to server, reason: %s\n\n", errorMsg); destroyRequest(pRequest); @@ -773,7 +781,7 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) { STscObj *pObj = pRequest->pTscObj; - char* db = getConnectionDB(pObj); + char* db = getDbOfConnection(pObj); if (db != NULL) { tstrncpy(pConnect->db, db, sizeof(pConnect->db)); } @@ -864,10 +872,10 @@ TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, c char userStr[TSDB_USER_LEN] = {0}; char passStr[TSDB_PASSWORD_LEN] = {0}; - strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); - strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + strncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen)); return taos_connect(ipStr, userStr, passStr, dbStr, port); } @@ -885,7 +893,7 @@ void* doFetchRow(SRequestObj* pRequest) { } SReqResultInfo* pResInfo = &pRequest->body.resInfo; - int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pResInfo->pData); + int32_t code = schedulerFetchRows(pRequest->body.pQueryJob, (void **)&pResInfo->pData); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; return NULL; @@ -1005,7 +1013,7 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t } } -char* getConnectionDB(STscObj* pObj) { +char* getDbOfConnection(STscObj* pObj) { char *p = NULL; pthread_mutex_lock(&pObj->mutex); size_t len = strlen(pObj->db); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index cbe6b90757..d62d7cb826 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -53,479 +53,481 @@ TEST(testCase, driverInit_Test) { // taos_init(); } -//TEST(testCase, connect_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// if (pConn == NULL) { -// printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); -// } -// taos_close(pConn); -//} -// -//TEST(testCase, create_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_account_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_account_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show users"); -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, drop_user_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_db_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show databases"); -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_close(pConn); -//} -// -//TEST(testCase, create_db_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create database abc1 vgroups 4"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_close(pConn); -//} -// -//TEST(testCase, create_dnode_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); -// if (taos_errno(pRes) != 0) { -// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// taos_close(pConn); -//} -// -//TEST(testCase, drop_dnode_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); -// if (taos_errno(pRes) != 0) { -// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// pRes = taos_query(pConn, "drop dnode 4"); -// if (taos_errno(pRes) != 0) { -// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, use_db_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in use db, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_close(pConn); -//} -// -// TEST(testCase, drop_db_test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// showDB(pConn); -// -// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// showDB(pConn); -// -// pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); -// } -// -// pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// pRes = taos_query(pConn, "drop stable `123_$^)`"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_close(pConn); -//} -// -//TEST(testCase, create_table_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); +TEST(testCase, connect_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); + } + taos_close(pConn); +} - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); +TEST(testCase, create_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); - //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); - //ASSERT_EQ(taos_errno(pRes), 0); + TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } - //taos_free_result(pRes); + taos_free_result(pRes); + taos_close(pConn); +} - //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); - //ASSERT_NE(taos_errno(pRes), 0); +TEST(testCase, create_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); - //taos_free_result(pRes); - //taos_close(pConn); -//} -// -//TEST(testCase, create_ctable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, show_vgroup_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show vgroups"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, create_multiple_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// taos_close(pConn); -// return; -// } -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// taos_free_result(pRes); -// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// -// for (int32_t i = 0; i < 20; ++i) { -// char sql[512] = {0}; -// snprintf(sql, tListLen(sql), -// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, -// (i + 1) * 30, (i + 2) * 40); -// TAOS_RES* pres = taos_query(pConn, sql); -// if (taos_errno(pres) != 0) { -// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); -// } -// taos_free_result(pres); -// } -// -// taos_close(pConn); -//} + TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } -//TEST(testCase, show_table_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); + taos_free_result(pRes); + taos_close(pConn); +} - //TAOS_RES* pRes = taos_query(pConn, "show tables"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //} +TEST(testCase, drop_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); - //pRes = taos_query(pConn, "show abc1.tables"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //} + TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); + taos_free_result(pRes); + taos_close(pConn); +} - //int32_t count = 0; - //char str[512] = {0}; +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%d: %s\n", ++count, str); - //} + TAOS_RES* pRes = taos_query(pConn, "show users"); + TAOS_ROW pRow = NULL; - //taos_free_result(pRes); - //taos_close(pConn); -//} + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show databases"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_close(pConn); +} + +TEST(testCase, create_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database abc1 vgroups 4"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_close(pConn); +} + +TEST(testCase, create_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); + if (taos_errno(pRes) != 0) { + printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); + if (taos_errno(pRes) != 0) { + printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + taos_close(pConn); +} + +TEST(testCase, drop_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + pRes = taos_query(pConn, "drop dnode 4"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, use_db_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} + + TEST(testCase, drop_db_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + showDB(pConn); + + TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + showDB(pConn); + + pRes = taos_query(pConn, "create database abc1"); + if (taos_errno(pRes) != 0) { + printf("create to drop db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("error in create stable, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + } + + pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + pRes = taos_query(pConn, "drop stable `123_$^)`"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); + } + + taos_close(pConn); +} + +TEST(testCase, create_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); + ASSERT_EQ(taos_errno(pRes), 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); + ASSERT_NE(taos_errno(pRes), 0); + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_ctable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != nullptr); + + TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); + if (taos_errno(pRes) != 0) { + printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_vgroup_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "show vgroups"); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_multiple_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + taos_close(pConn); + return; + } + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + taos_free_result(pRes); + pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + + for (int32_t i = 0; i < 20; ++i) { + char sql[512] = {0}; + snprintf(sql, tListLen(sql), + "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, + (i + 1) * 30, (i + 2) * 40); + TAOS_RES* pres = taos_query(pConn, sql); + if (taos_errno(pres) != 0) { + printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); + } + taos_free_result(pres); + } + + taos_close(pConn); +} + +TEST(testCase, show_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + taos_free_result(pRes); + + pRes = taos_query(pConn, "show abc1.tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + int32_t count = 0; + char str[512] = {0}; + + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%d: %s\n", ++count, str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != nullptr); + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); + if (taos_errno(pRes) != 0) { + printf("error in creating db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in using db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop stable st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, generated_request_id_test) { + SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + for (int32_t i = 0; i < 50000; ++i) { + uint64_t v = generateRequestId(); + void* result = taosHashGet(phash, &v, sizeof(v)); + if (result != nullptr) { + printf("0x%lx, index:%d\n", v, i); + } + assert(result == nullptr); + taosHashPut(phash, &v, sizeof(v), NULL, 0); + } + + taosHashCleanup(phash); +} -//TEST(testCase, drop_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in using db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "drop stable st1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// -//TEST(testCase, generated_request_id_test) { -// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); -// -// for (int32_t i = 0; i < 50000; ++i) { -// uint64_t v = generateRequestId(); -// void* result = taosHashGet(phash, &v, sizeof(v)); -// if (result != nullptr) { -// printf("0x%lx, index:%d\n", v, i); -// } -// assert(result == nullptr); -// taosHashPut(phash, &v, sizeof(v), NULL, 0); -// } -// -// taosHashCleanup(phash); -//} -// TEST(testCase, create_topic_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -546,23 +548,32 @@ TEST(testCase, create_topic_Test) { char* sql = "select * from tu"; pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, insert_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); if (taos_errno(pRes) != 0) { - printf("failed to create topic, reason:%s\n", taos_errstr(pRes)); - ASSERT_TRUE(0); + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); } taos_free_result(pRes); taos_close(pConn); } +#if 0 TEST(testCase, tmq_subscribe_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", "abc1", 0); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg1"); @@ -584,6 +595,7 @@ TEST(testCase, tmq_consume_Test) { TEST(testCase, tmq_commit_TEST) { } +#endif //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -603,56 +615,56 @@ TEST(testCase, tmq_commit_TEST) { // taos_close(pConn); //} -//TEST(testCase, projection_query_tables) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //ASSERT_NE(pConn, nullptr); +TEST(testCase, projection_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); - //pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); + pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); - //pRes = taos_query(pConn, "create table tu using st1 tags(1)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); - //for(int32_t i = 0; i < 100000; ++i) { - //char sql[512] = {0}; - //sprintf(sql, "insert into tu values(now+%da, %d)", i, i); - //TAOS_RES* p = taos_query(pConn, sql); - //if (taos_errno(p) != 0) { - //printf("failed to insert data, reason:%s\n", taos_errstr(p)); - //} + for(int32_t i = 0; i < 100000; ++i) { + char sql[512] = {0}; + sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } - //taos_free_result(p); - //} + taos_free_result(p); + } - //pRes = taos_query(pConn, "select * from tu"); - //if (taos_errno(pRes) != 0) { - //printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} + pRes = taos_query(pConn, "select * from tu"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } - //taos_free_result(pRes); - //taos_close(pConn); -//} + taos_free_result(pRes); + taos_close(pConn); +} //TEST(testCase, projection_query_stables) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 40e278f54a..7a0966df57 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -44,7 +44,7 @@ int32_t tsRpcTimer = 300; int32_t tsRpcMaxTime = 600; // seconds; int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; -int32_t tsMaxConnections = 5000; +int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 82f6a8e49a..73f755bf1a 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -630,7 +630,7 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { } } -#define SWAP(a, b, c) \ +#define TSWAP(a, b, c) \ do { \ typeof(a) __tmp = (a); \ (a) = (b); \ @@ -642,35 +642,35 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf switch (type) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: { - SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); + TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); break; } case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - SWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); + TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); break; } case TSDB_DATA_TYPE_DOUBLE: { - SWAP(*(double *)(pLeft), *(double *)(pRight), double); + TSWAP(*(double *)(pLeft), *(double *)(pRight), double); break; } case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: { - SWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); + TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); break; } case TSDB_DATA_TYPE_FLOAT: { - SWAP(*(float *)(pLeft), *(float *)(pRight), float); + TSWAP(*(float *)(pLeft), *(float *)(pRight), float); break; } case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: { - SWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); + TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); break; } diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index a2ca86438b..079c50f403 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -90,9 +90,9 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNCTION)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNCTION)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNCTION)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNC)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; @@ -119,6 +119,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { // Requests handled by VNODE pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; @@ -184,9 +185,13 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; rpcInit.ckey = INTERNAL_CKEY; - rpcInit.secret = INTERNAL_SECRET; + rpcInit.spi = 1; rpcInit.parent = pDnode; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + rpcInit.secret = pass; + pMgmt->clientRpc = rpcOpen(&rpcInit); if (pMgmt->clientRpc == NULL) { dError("failed to init rpc client"); @@ -261,20 +266,18 @@ static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRp static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (strcmp(user, INTERNAL_USER) == 0) { - // A simple temporary implementation - char pass[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 0; + *spi = 1; *encrypt = 0; *ckey = 0; return 0; } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - // A simple temporary implementation - char pass[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 0; + *spi = 1; *encrypt = 0; *ckey = 0; return 0; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 0d4b9c803d..df6cdc10cf 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -892,7 +892,7 @@ int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); if (pVnode == NULL) return -1; - int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, false); + int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, false); dndReleaseVnode(pDnode, pVnode); return code; } @@ -910,11 +910,11 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; int32_t maxFetchThreads = 4; - int32_t minFetchThreads = MIN(maxFetchThreads, pDnode->env.numOfCores); - int32_t minQueryThreads = MAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); + int32_t minFetchThreads = TMIN(maxFetchThreads, pDnode->env.numOfCores); + int32_t minQueryThreads = TMAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = MAX(pDnode->env.numOfCores, 1); - int32_t maxSyncThreads = MAX(pDnode->env.numOfCores / 2, 1); + int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1); + int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1); SWorkerPool *pPool = &pMgmt->queryPool; pPool->name = "vnode-query"; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 9f724faeb9..955f62610a 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -96,6 +96,15 @@ class Testbase { #define CheckBinary(val, len) \ { EXPECT_STREQ(test.GetShowBinary(len), val); } +#define CheckBinaryByte(b, len) \ + { \ + char* bytes = (char*)calloc(1, len); \ + for (int32_t i = 0; i < len - 1; ++i) { \ + bytes[i] = b; \ + } \ + EXPECT_STREQ(test.GetShowBinary(len), bytes); \ + } + #define CheckInt8(val) \ { EXPECT_EQ(test.GetShowInt8(), val); } diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index fd5dcd77d9..3d61db8268 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -27,8 +27,8 @@ void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; }; tsem_t* TestClient::GetSem() { return &sem; } bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { - char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); @@ -42,7 +42,7 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint rpcInit.ckey = (char*)"key"; rpcInit.parent = this; rpcInit.secret = (char*)secretEncrypt; - // rpcInit.spi = 1; + rpcInit.spi = 1; clientRpc = rpcOpen(&rpcInit); ASSERT(clientRpc); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 803dd453a1..02ce3a1591 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -314,7 +314,7 @@ typedef struct { int8_t outputType; int32_t outputLen; int32_t bufSize; - int64_t sigature; + int64_t signature; int32_t commentSize; int32_t codeSize; char* pComment; diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 3e9ab99a45..44a5836bd6 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAuth.h" +#include "mndUser.h" static int32_t mndProcessAuthReq(SMnodeMsg *pReq); @@ -25,7 +26,24 @@ int32_t mndInitAuth(SMnode *pMnode) { void mndCleanupAuth(SMnode *pMnode) {} -int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { return 0; } +int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SUserObj *pUser = mndAcquireUser(pMnode, user); + if (pUser == NULL) { + *secret = 0; + mError("user:%s, failed to auth user since %s", user, terrstr()); + return -1; + } + + *spi = 1; + *encrypt = 0; + *ckey = 0; + + memcpy(secret, pUser->pass, TSDB_PASSWORD_LEN); + mndReleaseUser(pMnode, pUser); + + mDebug("user:%s, auth info is returned", user); + return 0; +} static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { SAuthReq *pAuth = pReq->rpcMsg.pCont; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 4bc570c11d..efdacf957b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -234,7 +234,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { } bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { - int64_t interval = ABS(pDnode->lastAccessTime - curMs); + int64_t interval = TABS(pDnode->lastAccessTime - curMs); if (interval > 3500 * pMnode->cfg.statusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index d406247bc1..f9dde7bc75 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -25,14 +25,14 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc); static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); -static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc); -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate); -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc); -static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg); -static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg); -static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew); +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate); +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc); +static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq); +static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq); +static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq); +static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); int32_t mndInitFunc(SMnode *pMnode) { @@ -44,13 +44,13 @@ int32_t mndInitFunc(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndFuncActionUpdate, .deleteFp = (SdbDeleteFp)mndFuncActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNCTION, mndProcessCreateFuncMsg); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNCTION, mndProcessDropFuncMsg); - mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_FUNCTION, mndProcessRetrieveFuncMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNC, mndProcessCreateFuncReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNC, mndProcessDropFuncReq); + mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_FUNC, mndProcessRetrieveFuncReq); - mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndGetFuncMeta); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndRetrieveFuncs); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndCancelGetNextFunc); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndGetFuncMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndRetrieveFuncs); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndCancelGetNextFunc); return sdbSetTable(pMnode->pSdb, table); } @@ -73,7 +73,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { SDB_SET_INT8(pRaw, dataPos, pFunc->outputType, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->outputLen, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->bufSize, FUNC_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pFunc->sigature, FUNC_ENCODE_OVER) + SDB_SET_INT64(pRaw, dataPos, pFunc->signature, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, FUNC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_ENCODE_OVER) @@ -104,13 +104,11 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { goto FUNC_DECODE_OVER; } - int32_t size = sizeof(SFuncObj) + TSDB_FUNC_COMMENT_LEN + TSDB_FUNC_CODE_LEN; - SSdbRow *pRow = sdbAllocRow(size); + SSdbRow *pRow = sdbAllocRow(sizeof(SFuncObj)); if (pRow == NULL) goto FUNC_DECODE_OVER; SFuncObj *pFunc = sdbGetRowObj(pRow); if (pFunc == NULL) goto FUNC_DECODE_OVER; - char *tmp = (char *)pFunc + sizeof(SFuncObj); int32_t dataPos = 0; SDB_GET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN, FUNC_DECODE_OVER) @@ -121,12 +119,18 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pFunc->outputType, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->outputLen, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->bufSize, FUNC_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pFunc->sigature, FUNC_DECODE_OVER) + SDB_GET_INT64(pRaw, dataPos, &pFunc->signature, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, FUNC_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pFunc->pData, pFunc->commentSize + pFunc->codeSize, FUNC_DECODE_OVER) - pFunc->pComment = pFunc->pData; - pFunc->pCode = (pFunc->pData + pFunc->commentSize); + + pFunc->pComment = calloc(1, pFunc->commentSize); + pFunc->pCode = calloc(1, pFunc->codeSize); + if (pFunc->pComment == NULL || pFunc->pCode == NULL) { + goto FUNC_DECODE_OVER; + } + + SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_DECODE_OVER) terrno = 0; @@ -148,136 +152,136 @@ static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc) { static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { mTrace("func:%s, perform delete action, row:%p", pFunc->name, pFunc); + tfree(pFunc->pCode); + tfree(pFunc->pComment); return 0; } -static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc) { - mTrace("func:%s, perform update action, old row:%p new row:%p", pOldFunc->name, pOldFunc, pNewFunc); +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { + mTrace("func:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); return 0; } -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate) { - SFuncObj *pFunc = calloc(1, sizeof(SFuncObj) + pCreate->commentSize + pCreate->codeSize); - pFunc->createdTime = taosGetTimestampMs(); - pFunc->funcType = pCreate->funcType; - pFunc->scriptType = pCreate->scriptType; - pFunc->outputType = pCreate->outputType; - pFunc->outputLen = pCreate->outputLen; - pFunc->bufSize = pCreate->bufSize; - pFunc->sigature = pCreate->sigature; - pFunc->commentSize = pCreate->commentSize; - pFunc->codeSize = pCreate->codeSize; - pFunc->pComment = pFunc->pData; - memcpy(pFunc->pComment, pCreate->pCont, pCreate->commentSize); - pFunc->pCode = pFunc->pData + pCreate->commentSize; - memcpy(pFunc->pCode, pCreate->pCont + pCreate->commentSize, pFunc->codeSize); - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); - if (pTrans == NULL) { - free(pFunc); - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; +static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) { + SSdb *pSdb = pMnode->pSdb; + SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName); + if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; } + return pFunc; +} + +static void mndReleaseFunc(SMnode *pMnode, SFuncObj *pFunc) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pFunc); +} + +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate) { + int32_t code = -1; + STrans *pTrans = NULL; + + SFuncObj func = {0}; + memcpy(func.name, pCreate->name, TSDB_FUNC_NAME_LEN); + func.createdTime = taosGetTimestampMs(); + func.funcType = pCreate->funcType; + func.scriptType = pCreate->scriptType; + func.outputType = pCreate->outputType; + func.outputLen = pCreate->outputLen; + func.bufSize = pCreate->bufSize; + func.signature = pCreate->signature; + func.commentSize = pCreate->commentSize; + func.codeSize = pCreate->codeSize; + func.pComment = malloc(func.commentSize); + func.pCode = malloc(func.codeSize); + if (func.pCode == NULL || func.pCode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto CREATE_FUNC_OVER; + } + + memcpy(func.pComment, pCreate->pCont, pCreate->commentSize); + memcpy(func.pCode, pCreate->pCont + pCreate->commentSize, func.codeSize); + + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); + if (pTrans == NULL) goto CREATE_FUNC_OVER; mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); - SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - free(pFunc); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto CREATE_FUNC_OVER; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto CREATE_FUNC_OVER; - SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - free(pFunc); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto CREATE_FUNC_OVER; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto CREATE_FUNC_OVER; - SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - free(pFunc); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto CREATE_FUNC_OVER; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto CREATE_FUNC_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_FUNC_OVER; - free(pFunc); + code = 0; + +CREATE_FUNC_OVER: + free(func.pCode); + free(func.pComment); mndTransDrop(pTrans); - return 0; + return code; } -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("func:%s, failed to drop since %s", pFunc->name, terrstr()); - return -1; - } +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); + if (pTrans == NULL) goto DROP_FUNC_OVER; + mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto DROP_FUNC_OVER; sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto DROP_FUNC_OVER; sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto DROP_FUNC_OVER; sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_FUNC_OVER; + code = 0; + +DROP_FUNC_OVER: mndTransDrop(pTrans); - return 0; + return code; } -static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; - SCreateFuncReq *pCreate = pMsg->rpcMsg.pCont; + SCreateFuncReq *pCreate = pReq->rpcMsg.pCont; pCreate->outputLen = htonl(pCreate->outputLen); pCreate->bufSize = htonl(pCreate->bufSize); - pCreate->sigature = htobe64(pCreate->sigature); + pCreate->signature = htobe64(pCreate->signature); pCreate->commentSize = htonl(pCreate->commentSize); pCreate->codeSize = htonl(pCreate->codeSize); mDebug("func:%s, start to create", pCreate->name); - SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pCreate->name); + SFuncObj *pFunc = mndAcquireFunc(pMnode, pCreate->name); if (pFunc != NULL) { - sdbRelease(pMnode->pSdb, pFunc); - terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + mndReleaseFunc(pMnode, pFunc); + if (pCreate->igExists) { + mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); + return 0; + } else { + terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + } else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) { + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } @@ -305,14 +309,13 @@ static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { return -1; } - if (pCreate->bufSize < 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { + if (pCreate->bufSize <= 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; mError("func:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } - int32_t code = mndCreateFunc(pMnode, pMsg, pCreate); - + int32_t code = mndCreateFunc(pMnode, pReq, pCreate); if (code != 0) { mError("func:%s, failed to create since %s", pCreate->name, terrstr()); return -1; @@ -321,9 +324,9 @@ static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SDropFuncReq *pDrop = pMsg->rpcMsg.pCont; +static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SDropFuncReq *pDrop = pReq->rpcMsg.pCont; mDebug("func:%s, start to drop", pDrop->name); @@ -333,14 +336,20 @@ static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { return -1; } - SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pDrop->name); + SFuncObj *pFunc = mndAcquireFunc(pMnode, pDrop->name); if (pFunc == NULL) { - terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; - mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); - return -1; + if (pDrop->igNotExists) { + mDebug("func:%s, not exist, ignore not exist is set", pDrop->name); + return 0; + } else { + terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; + mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } } - int32_t code = mndDropFunc(pMnode, pMsg, pFunc); + int32_t code = mndDropFunc(pMnode, pReq, pFunc); + mndReleaseFunc(pMnode, pFunc); if (code != 0) { mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); @@ -350,15 +359,26 @@ static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { + int32_t code = -1; + SMnode *pMnode = pReq->pMnode; - SRetrieveFuncReq *pRetrieve = pMsg->rpcMsg.pCont; + SRetrieveFuncReq *pRetrieve = pReq->rpcMsg.pCont; pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); + if (pRetrieve->numOfFuncs <= 0 || pRetrieve->numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { + terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE; + return -1; + } - int32_t size = sizeof(SRetrieveFuncRsp) + (sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN) * pRetrieve->numOfFuncs + 16384; + int32_t fsize = sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN + TSDB_FUNC_COMMENT_LEN; + int32_t size = sizeof(SRetrieveFuncRsp) + fsize * pRetrieve->numOfFuncs; SRetrieveFuncRsp *pRetrieveRsp = rpcMallocCont(size); + if (pRetrieveRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FUNC_RETRIEVE_OVER; + } + pRetrieveRsp->numOfFuncs = htonl(pRetrieve->numOfFuncs); char *pOutput = pRetrieveRsp->pFuncInfos; @@ -366,37 +386,42 @@ static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { char funcName[TSDB_FUNC_NAME_LEN] = {0}; memcpy(funcName, pRetrieve->pFuncNames + i * TSDB_FUNC_NAME_LEN, TSDB_FUNC_NAME_LEN); - SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, funcName); + SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); if (pFunc == NULL) { terrno = TSDB_CODE_MND_INVALID_FUNC; mError("func:%s, failed to retrieve since %s", funcName, terrstr()); - return -1; + goto FUNC_RETRIEVE_OVER; } SFuncInfo *pFuncInfo = (SFuncInfo *)pOutput; - - strncpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); + memcpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); pFuncInfo->funcType = pFunc->funcType; pFuncInfo->scriptType = pFunc->scriptType; pFuncInfo->outputType = pFunc->outputType; pFuncInfo->outputLen = htonl(pFunc->outputLen); pFuncInfo->bufSize = htonl(pFunc->bufSize); - pFuncInfo->sigature = htobe64(pFunc->sigature); + pFuncInfo->signature = htobe64(pFunc->signature); pFuncInfo->commentSize = htonl(pFunc->commentSize); pFuncInfo->codeSize = htonl(pFunc->codeSize); - memcpy(pFuncInfo->pCont, pFunc->pCode, pFunc->commentSize + pFunc->codeSize); - - pOutput += sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize; + memcpy(pFuncInfo->pCont, pFunc->pComment, pFunc->commentSize); + memcpy(pFuncInfo->pCont + pFunc->commentSize, pFunc->pCode, pFunc->codeSize); + pOutput += (sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize); + mndReleaseFunc(pMnode, pFunc); } - pMsg->pCont = pRetrieveRsp; - pMsg->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); + pReq->pCont = pRetrieveRsp; + pReq->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); - return 0; + code = 0; + +FUNC_RETRIEVE_OVER: + if (code != 0) rpcFreeCont(pRetrieveRsp); + + return code; } -static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -454,7 +479,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *p pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, "show funcs"); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -477,8 +502,8 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t le return tDataTypes[type].name; } -static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SFuncObj *pFunc = NULL; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 22fdfde2ac..76fabc96ce 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -261,7 +261,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); } - pConn->numOfQueries = MIN(QUERY_SAVE_SIZE, numOfQueries); + pConn->numOfQueries = TMIN(QUERY_SAVE_SIZE, numOfQueries); int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc); if (saveSize > 0 && pConn->pQueries != NULL) { diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 28fe0551c2..e2ddcee0e9 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -296,7 +296,7 @@ char *mndShowStr(int32_t showType) { return "show streamtables"; case TSDB_MGMT_TABLE_TP: return "show topics"; - case TSDB_MGMT_TABLE_FUNCTION: + case TSDB_MGMT_TABLE_FUNC: return "show functions"; default: return "undefined"; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 5ed801e3ea..45b8428663 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -123,8 +123,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { goto STB_DECODE_OVER; } - int32_t size = sizeof(SStbObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); - SSdbRow *pRow = sdbAllocRow(size); + SSdbRow *pRow = sdbAllocRow(sizeof(SStbObj)); if (pRow == NULL) goto STB_DECODE_OVER; SStbObj *pStb = sdbGetRowObj(pRow); @@ -143,6 +142,9 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; pStb->pSchema = calloc(totalCols, sizeof(SSchema)); + if (pStb->pSchema == NULL) { + goto STB_DECODE_OVER; + } for (int32_t i = 0; i < totalCols; ++i) { SSchema *pSchema = &pStb->pSchema[i]; @@ -448,7 +450,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCr stbObj.pSchema[i].colId = i + 1; } - int32_t code = 0; + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_STB_OVER; @@ -481,7 +483,7 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name); if (pStb != NULL) { - sdbRelease(pMnode->pSdb, pStb); + mndReleaseStb(pMnode, pStb); if (pCreate->igExists) { mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); return 0; @@ -492,6 +494,7 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { } } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; } // topic should have different name with stb @@ -640,7 +643,7 @@ static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pStb) { DROP_STB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { @@ -665,7 +668,6 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { mndReleaseStb(pMnode, pStb); if (code != 0) { - terrno = code; mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 035c05c286..aacaa0d728 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -56,7 +56,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) { int64_t last = walGetLastVer(pWal); mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); - first = MAX(lastSdbVer + 1, first); + first = TMAX(lastSdbVer + 1, first); for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) { if (walReadWithHandle(pHandle, ver) < 0) { mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 2301df65d7..03226c7400 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -390,9 +390,11 @@ static void mndTransDropActions(SArray *pArray) { } void mndTransDrop(STrans *pTrans) { - mndTransDropData(pTrans); - mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); - tfree(pTrans); + if (pTrans != NULL) { + mndTransDropData(pTrans); + mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); + tfree(pTrans); + } } static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index ad378953eb..c36e128893 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -60,9 +60,9 @@ void mndCleanupUser(SMnode *pMnode) {} static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN); - taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; @@ -202,7 +202,7 @@ SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); if (pUser == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + terrno = TSDB_CODE_MND_USER_NOT_EXIST; } return pUser; } @@ -214,9 +214,9 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pReq) { SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN); - taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; userObj.superUser = 0; @@ -351,8 +351,9 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) { SUserObj newUser = {0}; memcpy(&newUser, pUser, sizeof(SUserObj)); - memset(pUser->pass, 0, sizeof(pUser->pass)); - taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)pAlter->pass, strlen(pAlter->pass), pass); + memcpy(pUser->pass, pass, TSDB_PASSWORD_LEN); newUser.updateTime = taosGetTimestampMs(); int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pReq); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0bed943b60..04fedbb3ce 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -459,7 +459,7 @@ static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void int32_t *pNumOfVgroups = p3; if (pVgroup->dbUid == uid) { - *pReplica = MAX(*pReplica, pVgroup->replica); + *pReplica = TMAX(*pReplica, pVgroup->replica); (*pNumOfVgroups)++; } diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 3ca35d58a7..5f6f2f3b4f 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -12,3 +12,4 @@ add_subdirectory(dnode) add_subdirectory(mnode) add_subdirectory(db) add_subdirectory(stb) +add_subdirectory(func) diff --git a/source/dnode/mnode/impl/test/func/CMakeLists.txt b/source/dnode/mnode/impl/test/func/CMakeLists.txt new file mode 100644 index 0000000000..26b0a60968 --- /dev/null +++ b/source/dnode/mnode/impl/test/func/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. FUNC_SRC) +add_executable(mnode_test_func ${FUNC_SRC}) +target_link_libraries( + mnode_test_func + PUBLIC sut +) + +add_test( + NAME mnode_test_func + COMMAND mnode_test_func +) diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp new file mode 100644 index 0000000000..9d31ec7314 --- /dev/null +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -0,0 +1,521 @@ +/** + * @file func.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module func tests + * @version 1.0 + * @date 2022-01-24 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "sut.h" + +class MndTestFunc : public ::testing::Test { + protected: + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_func", 9038); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} +}; + +Testbase MndTestFunc::test; + +TEST_F(MndTestFunc, 01_Show_Func) { + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, PATH_MAX + VARSTR_HEADER_SIZE, "comment"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_INT, 4, "aggregate"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE, "outputtype"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_INT, 4, "code_len"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_INT, 4, "bufsize"); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(MndTestFunc, 02_Create_Func) { + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, ""); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN + 1); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN - 1); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq) + 24; + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); + pReq->pCont[0] = 0; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq) + 24; + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); + pReq->pCont[0] = 'a'; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq) + 24; + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); + pReq->pCont[0] = 'a'; + pReq->bufSize = htonl(TSDB_FUNC_BUF_SIZE + 1); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); + } + + for (int32_t i = 0; i < 3; ++i) { + int32_t contLen = sizeof(SCreateFuncReq); + int32_t commentSize = TSDB_FUNC_COMMENT_LEN; + int32_t codeSize = TSDB_FUNC_CODE_LEN; + contLen = (contLen + codeSize + commentSize); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->igExists = 0; + if (i == 2) pReq->igExists = 1; + pReq->funcType = 1; + pReq->scriptType = 2; + pReq->outputType = TSDB_DATA_TYPE_SMALLINT; + pReq->outputLen = htonl(12); + pReq->bufSize = htonl(4); + pReq->signature = htobe64(5); + pReq->commentSize = htonl(commentSize); + pReq->codeSize = htonl(codeSize); + for (int32_t i = 0; i < commentSize - 1; ++i) { + pReq->pCont[i] = 'm'; + } + for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { + pReq->pCont[i] = 'd'; + } + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (i == 0 || i == 2) { + ASSERT_EQ(pRsp->code, 0); + } else { + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_ALREADY_EXIST); + } + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("f1", TSDB_FUNC_NAME_LEN); + CheckBinaryByte('m', TSDB_FUNC_COMMENT_LEN); + CheckInt32(0); + CheckBinary("SMALLINT", TSDB_TYPE_STR_MAX_LEN); + CheckTimestamp(); + CheckInt32(TSDB_FUNC_CODE_LEN); + CheckInt32(4); +} + +TEST_F(MndTestFunc, 03_Retrieve_Func) { + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(1); + strcpy(pReq->pFuncNames, "f1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; + pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + + EXPECT_STREQ(pFuncInfo->name, "f1"); + EXPECT_EQ(pFuncInfo->funcType, 1); + EXPECT_EQ(pFuncInfo->scriptType, 2); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pFuncInfo->outputLen, 12); + EXPECT_EQ(pFuncInfo->bufSize, 4); + EXPECT_EQ(pFuncInfo->signature, 5); + EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); + EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { + comments[i] = 'm'; + } + char codes[TSDB_FUNC_CODE_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { + codes[i] = 'd'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 0; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_RETRIEVE); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_RETRIEVE); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + strcpy(pReq->pFuncNames, "f2"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + int32_t commentSize = 1024; + int32_t codeSize = 9527; + contLen = (contLen + codeSize + commentSize); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f2"); + pReq->igExists = 1; + pReq->funcType = 2; + pReq->scriptType = 3; + pReq->outputType = TSDB_DATA_TYPE_BINARY; + pReq->outputLen = htonl(24); + pReq->bufSize = htonl(6); + pReq->signature = htobe64(18); + pReq->commentSize = htonl(commentSize); + pReq->codeSize = htonl(codeSize); + for (int32_t i = 0; i < commentSize - 1; ++i) { + pReq->pCont[i] = 'p'; + } + for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { + pReq->pCont[i] = 'q'; + } + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(1); + strcpy(pReq->pFuncNames, "f2"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; + pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + + EXPECT_STREQ(pFuncInfo->name, "f2"); + EXPECT_EQ(pFuncInfo->funcType, 2); + EXPECT_EQ(pFuncInfo->scriptType, 3); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pFuncInfo->outputLen, 24); + EXPECT_EQ(pFuncInfo->bufSize, 6); + EXPECT_EQ(pFuncInfo->signature, 18); + EXPECT_EQ(pFuncInfo->commentSize, 1024); + EXPECT_EQ(pFuncInfo->codeSize, 9527); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char* comments = (char*)calloc(1, 1024); + for (int32_t i = 0; i < 1024 - 1; ++i) { + comments[i] = 'p'; + } + char* codes = (char*)calloc(1, 9527); + for (int32_t i = 0; i < 9527 - 1; ++i) { + codes[i] = 'q'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + free(comments); + free(codes); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 2; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + strcpy(pReq->pFuncNames, "f2"); + strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; + pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + + { + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + + EXPECT_STREQ(pFuncInfo->name, "f2"); + EXPECT_EQ(pFuncInfo->funcType, 2); + EXPECT_EQ(pFuncInfo->scriptType, 3); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pFuncInfo->outputLen, 24); + EXPECT_EQ(pFuncInfo->bufSize, 6); + EXPECT_EQ(pFuncInfo->signature, 18); + EXPECT_EQ(pFuncInfo->commentSize, 1024); + EXPECT_EQ(pFuncInfo->codeSize, 9527); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char* comments = (char*)calloc(1, 1024); + for (int32_t i = 0; i < 1024 - 1; ++i) { + comments[i] = 'p'; + } + char* codes = (char*)calloc(1, 9527); + for (int32_t i = 0; i < 9527 - 1; ++i) { + codes[i] = 'q'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + free(comments); + free(codes); + } + + { + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos + sizeof(SFuncInfo) + 1024 + 9527); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + EXPECT_STREQ(pFuncInfo->name, "f1"); + EXPECT_EQ(pFuncInfo->funcType, 1); + EXPECT_EQ(pFuncInfo->scriptType, 2); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pFuncInfo->outputLen, 12); + EXPECT_EQ(pFuncInfo->bufSize, 4); + EXPECT_EQ(pFuncInfo->signature, 5); + EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); + EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { + comments[i] = 'm'; + } + char codes[TSDB_FUNC_CODE_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { + codes[i] = 'd'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + } + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 2; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + strcpy(pReq->pFuncNames, "f2"); + strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f3"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + } +} + +TEST_F(MndTestFunc, 04_Drop_Func) { + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, ""); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME); + } + + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f3"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST); + } + + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f3"); + pReq->igNotExists = 1; + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + // restart + test.Restart(); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("f2", TSDB_FUNC_NAME_LEN); +} diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index a9267b0ea3..1f415d3e4b 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -152,6 +152,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (insertFp != NULL) { code = (*insertFp)(pSdb, pRow->pObj); if (code != 0) { + code = terrno; taosWLockLatch(pLock); taosHashRemove(hash, pRow->pObj, keySize); taosWUnLockLatch(pLock); @@ -162,10 +163,10 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { - pSdb->maxId[pRow->type] = MAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); + pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); } if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT64) { - pSdb->maxId[pRow->type] = MAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); + pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); } pSdb->tableVer[pRow->type]++; @@ -420,13 +421,13 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { while (ppRow != NULL) { SSdbRow *pRow = *ppRow; int32_t id = *(int32_t *)pRow->pObj; - maxId = MAX(id, maxId); + maxId = TMAX(id, maxId); ppRow = taosHashIterate(hash, ppRow); } taosRUnLockLatch(pLock); - maxId = MAX(maxId, pSdb->maxId[type]); + maxId = TMAX(maxId, pSdb->maxId[type]); return maxId + 1; } diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index d8ee48ff23..dfdc144750 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -587,7 +587,11 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { pBuf = value.data; metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { + free(tbCfg.name); + free(tbCfg.stbCfg.pTagSchema); continue; + } else if (tbCfg.type == META_CHILD_TABLE) { + kvRowFree(tbCfg.ctbCfg.pTag); } return tbCfg.name; } else { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1034a05443..a3a3804562 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -81,6 +81,10 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl } void tqClose(STQ* pTq) { + if (pTq) { + tfree(pTq->path); + free(pTq); + } // TODO } diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c index 59b680ec15..80b59d8756 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -459,7 +459,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { while (true) { if (pReadh->pDCols[0]->numOfRows - ridx == 0) break; - int rowsToMerge = MIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows); + int rowsToMerge = TMIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows); tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 135b81f282..be46f63168 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -359,7 +359,7 @@ void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd) { tsdbResetFSStatus(pfs->nstatus); pfs->nstatus->meta = pfs->cstatus->meta; // if (pfs->cstatus->pmf == NULL) { - pfs->nstatus->meta.version = 0; + pfs->nstatus->meta.version += 1; // } else { // pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1; // } @@ -836,7 +836,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // return -1; // } -// maxBufSize = MAX(maxBufSize, rInfo.size); +// maxBufSize = TMAX(maxBufSize, rInfo.size); // if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) { // tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index b16b3581df..81f0a6736b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -153,8 +153,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (fKey == INT64_MAX && rowKey == INT64_MAX) break; if (fKey < rowKey) { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); filterIter++; if (filterIter >= nFilterKeys) { @@ -170,8 +170,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsInserted++; pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } @@ -196,12 +196,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsUpdated++; pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } else { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); } } @@ -714,8 +714,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (fKey == INT64_MAX && rowKey == INT64_MAX) break; if (fKey < rowKey) { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); filterIter++; if (filterIter >= nFilterKeys) { @@ -731,8 +731,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsInserted++; pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } @@ -757,12 +757,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsUpdated++; pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } else { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 15748118d7..68c7f190db 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -504,7 +504,7 @@ void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { if (emptyQueryTimewindow(pTsdbReadHandle)) { if (pCond->order != pTsdbReadHandle->order) { pTsdbReadHandle->order = pCond->order; - SWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); + TSWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); } return; @@ -971,7 +971,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { pHandle->cur.mixBlock = true; if (!ASCENDING_TRAVERSE(pHandle->order)) { - SWAP(win->skey, win->ekey, TSKEY); + TSWAP(win->skey, win->ekey, TSKEY); } return true; @@ -1074,8 +1074,8 @@ static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, i assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } - s = MIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); - e = MAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); + s = TMIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); + e = TMAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); // discard the unqualified data block based on the query time window int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); @@ -1240,7 +1240,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* // update the last key value pCheckInfo->lastKey = cur->win.ekey + step; if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - SWAP(cur->win.skey, cur->win.ekey, TSKEY); + TSWAP(cur->win.skey, cur->win.ekey, TSKEY); } cur->mixBlock = true; @@ -1795,7 +1795,7 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa int32_t end = endPos; if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - SWAP(start, end, int32_t); + TSWAP(start, end, int32_t); } assert(pTsdbReadHandle->outputCapacity >= (end - start + 1)); @@ -2020,7 +2020,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - SWAP(cur->win.skey, cur->win.ekey, TSKEY); + TSWAP(cur->win.skey, cur->win.ekey, TSKEY); } moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index f49babb1a5..14b9a5124f 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -68,7 +68,7 @@ void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { void * ptr; if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { - uint64_t capacity = MAX(pVMA->ssize, size); + uint64_t capacity = TMAX(pVMA->ssize, size); pNode = vArenaNodeNew(capacity); if (pNode == NULL) { // TODO: handle error @@ -114,4 +114,4 @@ static void vArenaNodeFree(SVArenaNode *pNode) { if (pNode) { free(pNode); } -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 8df6b42566..434498eef5 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -71,6 +71,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { void vnodeCloseBufPool(SVnode *pVnode) { if (pVnode->pBufPool) { + tfree(pVnode->pBufPool->pMAF); vmaDestroy(pVnode->pBufPool->inuse); while (true) { diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 730155b75a..6829d778f0 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -119,6 +119,7 @@ static void* loop(void* arg) { pthread_mutex_unlock(&(vnodeMgr.mutex)); (*(pTask->execute))(pTask->arg); + free(pTask); } return NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 4012914723..f619499c8d 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -25,7 +25,7 @@ int vnodeQueryOpen(SVnode *pVnode) { } int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vTrace("query message is processing"); + vTrace("message in query queue is processing"); switch (pMsg->msgType) { case TDMT_VND_QUERY: @@ -39,7 +39,7 @@ int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vTrace("fetch message is processed"); + vTrace("message in fetch queue is processing"); switch (pMsg->msgType) { case TDMT_VND_FETCH: return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg); @@ -145,6 +145,15 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { _exit: + free(pSW->pSchema); + free(pSW); + free(pTbCfg->name); + free(pTbCfg); + if (pTbCfg->type == META_SUPER_TABLE) { + free(pTbCfg->stbCfg.pTagSchema); + } else if (pTbCfg->type == META_SUPER_TABLE) { + kvRowFree(pTbCfg->ctbCfg.pTag); + } rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; rpcMsg.pCont = pTbMetaMsg; @@ -156,8 +165,8 @@ _exit: return 0; } -static void freeItemHelper(void* pItem) { - char* p = *(char**)pItem; +static void freeItemHelper(void *pItem) { + char *p = *(char **)pItem; free(p); } @@ -187,14 +196,14 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { // TODO: temp debug, and should del when show tables command ok vInfo("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); if (numOfTables > 10000) { - numOfTables = 10000; + numOfTables = 10000; } metaCloseTbCursor(pCur); int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; - //int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); + // int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); int32_t payloadLen = rowLen * numOfTables; // SVShowTablesFetchReq *pFetchReq = pMsg->pCont; @@ -208,6 +217,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { STR_TO_VARSTR(p, n); p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); + // free(n); } pFetchRsp->numOfRows = htonl(numOfTables); @@ -222,6 +232,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); + taosArrayDestroyEx(pArray, freeItemHelper); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 211bb9cc4b..4993e8af23 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); @@ -53,7 +53,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vCreateTbReq; SVCreateTbBatchReq vCreateTbBatchReq; - void *ptr = vnodeMalloc(pVnode, pMsg->contLen); + void * ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -76,6 +76,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // TODO: maybe need to clear the requst struct + free(vCreateTbReq.stbCfg.pSchema); + free(vCreateTbReq.stbCfg.pTagSchema); + free(vCreateTbReq.name); break; case TDMT_VND_CREATE_TABLE: tSVCreateTbBatchReqDeserialize(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); @@ -130,7 +133,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } } - + return 0; } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 43b8ae53a4..2197fdfd62 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -86,7 +86,7 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("error rsp for use db, code:%x, db:%s", rpcRsp.code, input->db); + ctgError("error rsp for use db, code:%s, db:%s", tstrerror(rpcRsp.code), input->db); CTG_ERR_RET(rpcRsp.code); } @@ -258,7 +258,7 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransport return TSDB_CODE_SUCCESS; } - ctgError("error rsp for stablemeta from mnode, code:%x, tbName:%s", rpcRsp.code, tbFullName); + ctgError("error rsp for stablemeta from mnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tbFullName); CTG_ERR_RET(rpcRsp.code); } @@ -320,18 +320,17 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, return TSDB_CODE_SUCCESS; } - ctgError("error rsp for table meta from vnode, code:%x, tbName:%s", rpcRsp.code, tNameGetTableName(pTableName)); + ctgError("error rsp for table meta from vnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tNameGetTableName(pTableName)); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%x, tbName:%s", code, tNameGetTableName(pTableName)); + ctgError("Process vnode tablemeta rsp failed, code:%s, tbName:%s", tstrerror(code), tNameGetTableName(pTableName)); CTG_ERR_RET(code); } ctgDebug("Got table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4ce652f5fd..bfd1bce499 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -304,7 +304,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO idata.info.bytes = pExpr[i].base.resSchema.bytes; idata.info.colId = pExpr[i].base.resSchema.colId; - int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform taosArrayPush(res->pDataBlock, &idata); } @@ -329,7 +329,7 @@ SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) { idata.info.bytes = pExpr->base.resSchema.bytes; idata.info.colId = pExpr->base.resSchema.colId; - int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform taosArrayPush(res->pDataBlock, &idata); } @@ -2679,7 +2679,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool pQueryAttr->order.order = TSDB_ORDER_ASC; if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->needReverseScan = false; @@ -2689,7 +2689,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { pQueryAttr->order.order = TSDB_ORDER_ASC; if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->needReverseScan = false; @@ -2700,7 +2700,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { //qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } pQueryAttr->order.order = TSDB_ORDER_ASC; @@ -2713,7 +2713,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool //qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, // pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2724,7 +2724,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool //qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, // pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2739,7 +2739,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool //qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, // pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2750,7 +2750,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool //qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, // pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); doUpdateLastKey(pQueryAttr); } @@ -2790,8 +2790,8 @@ static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, in static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; - TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); if (QUERY_IS_ASC_QUERY(pQueryAttr)) { getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); @@ -3474,7 +3474,7 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) return; } - SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; SWITCH_ORDER(pTableQueryInfo->cur.order); @@ -4096,8 +4096,8 @@ void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { */ STimeWindow w = TSWINDOW_INITIALIZER; - TSKEY sk = MIN(win.skey, win.ekey); - TSKEY ek = MAX(win.skey, win.ekey); + TSKEY sk = TMIN(win.skey, win.ekey); + TSKEY ek = TMAX(win.skey, win.ekey); getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4222,7 +4222,7 @@ static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SQLFunctio } // SResultRowEntryInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); -// pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); +// pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); } } } @@ -5302,7 +5302,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* rpcInit.user = (char *)"root"; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.ckey = "key"; -// rpcInit.spi = 1; + rpcInit.spi = 1; rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; pInfo->pTransporter = rpcOpen(&rpcInit); @@ -7204,8 +7204,8 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); STimeWindow w = TSWINDOW_INITIALIZER; - TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); pInfo->pFillInfo = diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index cad40b342a..ff2fb95328 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -443,7 +443,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI pInfo->intermediateBytes = (int16_t)sizeof(SPercentileInfo); } else if (functionId == FUNCTION_LEASTSQR) { pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = MAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string + pInfo->bytes = TMAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string pInfo->intermediateBytes = pInfo->bytes; } else if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) { pInfo->type = TSDB_DATA_TYPE_BINARY; @@ -1331,7 +1331,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { continue; \ } \ (num) += 1; \ - (r) += POW2(((type *)d)[i] - (delta)); \ + (r) += TPOW2(((type *)d)[i] - (delta)); \ } static void stddev_function(SQLFunctionCtx *pCtx) { @@ -1370,7 +1370,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { continue; } num += 1; - *retVal += POW2(((int32_t *)pData)[i] - avg); + *retVal += TPOW2(((int32_t *)pData)[i] - avg); } break; } @@ -1484,7 +1484,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { continue; } num += 1; - *retVal += POW2(((int32_t *)pData)[i] - avg); + *retVal += TPOW2(((int32_t *)pData)[i] - avg); } break; } @@ -4210,8 +4210,8 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock pDist->totalRows += pSrc->totalRows; if (pResInfo->hasResult == DATA_SET_FLAG) { - pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows); - pDist->minRows = MIN(pDist->minRows, pSrc->minRows); + pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows); + pDist->minRows = TMIN(pDist->minRows, pSrc->minRows); } else { pDist->maxRows = pSrc->maxRows; pDist->minRows = pSrc->minRows; diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/function/src/tbinoperator.c index 1803e282bf..4de11cbe71 100644 --- a/source/libs/function/src/tbinoperator.c +++ b/source/libs/function/src/tbinoperator.c @@ -26,7 +26,7 @@ void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight int32_t *pRight = (int32_t *)right; double * pOutput = (double *)output; - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : TMAX(numLeft, numRight) - 1; int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; if (numLeft == numRight) { @@ -181,7 +181,7 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { } void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double *output=(double*)out; @@ -220,7 +220,7 @@ void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int } void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double *output=(double*)out; @@ -257,7 +257,7 @@ void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int } } void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double *output=(double*)out; @@ -296,7 +296,7 @@ void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out } void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double *output=(double*)out; @@ -342,7 +342,7 @@ void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, } void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; double * output = (double *)out; @@ -416,7 +416,7 @@ void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *ou void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { int32_t len = pLeft->bytes + pRight->bytes; - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; char *output = (char *)out; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 5299a7dc5f..46b4c9d7c6 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -152,7 +152,7 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, uint64_t addPrefix = 0; uint8_t* data = fstSliceData(s, NULL); if (t && t->inp == data[i]) { - uint64_t commPrefix = MIN(t->out, *out); + uint64_t commPrefix = TMIN(t->out, *out); uint64_t tAddPrefix = t->out - commPrefix; (*out) = (*out) - commPrefix; t->out = commPrefix; @@ -244,8 +244,8 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil bool anyOuts = (node->finalOutput != 0); for (size_t i = 0; i < sz; i++) { FstTransition* t = taosArrayGet(node->trans, i); - tSize = MAX(tSize, packDeltaSize(addr, t->addr)); - oSize = MAX(oSize, packSize(t->out)); + tSize = TMAX(tSize, packDeltaSize(addr, t->addr)); + oSize = TMAX(oSize, packSize(t->out)); anyOuts = anyOuts || (t->out != 0); } diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index cd84222c65..f53c5ea279 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -171,6 +171,8 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); void destroyBoundColumnInfo(SParsedDataColInfo* pColList); void destroyBlockArrayList(SArray* pDataBlockList); +void destroyBlockHashmap(SHashObj* pDataBlockHash); + int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 6e91ad997c..410e4fb187 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -65,7 +65,7 @@ program ::= cmd. {} //////////////////////////////////THE SHOW STATEMENT/////////////////////////////////////////// cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} -cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} +cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);} cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 8327b7c131..0a2f2b20f2 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -213,10 +213,11 @@ SQueryStmtInfo *createQueryInfo() { pQueryInfo->slimit.limit = -1; pQueryInfo->slimit.offset = 0; - pQueryInfo->pDownstream = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->pDownstream = taosArrayInit(4, POINTER_BYTES); pQueryInfo->window = TSWINDOW_INITIALIZER; pQueryInfo->exprList = calloc(10, POINTER_BYTES); + for(int32_t i = 0; i < 10; ++i) { pQueryInfo->exprList[i] = taosArrayInit(4, POINTER_BYTES); } @@ -232,7 +233,8 @@ static void destroyQueryInfoImpl(SQueryStmtInfo* pQueryInfo) { cleanupFieldInfo(&pQueryInfo->fieldsInfo); dropAllExprInfo(pQueryInfo->exprList, 10); - pQueryInfo->exprList = NULL; + + tfree(pQueryInfo->exprList); columnListDestroy(pQueryInfo->colList); pQueryInfo->colList = NULL; @@ -258,10 +260,10 @@ void destroyQueryInfo(SQueryStmtInfo* pQueryInfo) { size_t numOfUpstream = taosArrayGetSize(pQueryInfo->pDownstream); for (int32_t i = 0; i < numOfUpstream; ++i) { - SQueryStmtInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pDownstream, i); - destroyQueryInfoImpl(pUpQueryInfo); - clearAllTableMetaInfo(pUpQueryInfo, false, 0); - tfree(pUpQueryInfo); + SQueryStmtInfo* pDownstream = taosArrayGetP(pQueryInfo->pDownstream, i); + destroyQueryInfoImpl(pDownstream); + clearAllTableMetaInfo(pDownstream, false, 0); + tfree(pDownstream); } destroyQueryInfoImpl(pQueryInfo); @@ -1267,7 +1269,7 @@ static int32_t checkFillQueryRange(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) // return buildInvalidOperationMsg(pMsgBuf, msg1); // } - int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); + int64_t timeRange = TABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); int64_t intervalRange = 0; if (!TIME_IS_VAR_DURATION(pQueryInfo->interval.intervalUnit)) { @@ -1353,7 +1355,7 @@ int32_t validateFillNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf numOfFillVal = numOfFields; } } else { - numOfFillVal = MIN(num, numOfFields); + numOfFillVal = TMIN(num, numOfFields); } int32_t j = 1; @@ -1395,6 +1397,13 @@ int32_t validateFillNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf static void pushDownAggFuncExprInfo(SQueryStmtInfo* pQueryInfo); static void addColumnNodeFromLowerLevel(SQueryStmtInfo* pQueryInfo); +static void freeItemHelper(void* pItem) { + void** p = pItem; + if (*p != NULL) { + tfree(*p); + } +} + int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { assert(pSqlNode != NULL && (pSqlNode->from == NULL || taosArrayGetSize(pSqlNode->from->list) > 0)); @@ -1590,7 +1599,10 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* SArray* functionList = extractFunctionList(pQueryInfo->exprList[i]); extractFunctionDesc(functionList, &pQueryInfo->info); - if ((code = checkForInvalidExpr(pQueryInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) { + code = checkForInvalidExpr(pQueryInfo, pMsgBuf); + taosArrayDestroyEx(functionList, freeItemHelper); + + if (code != TSDB_CODE_SUCCESS) { return code; } } @@ -1928,7 +1940,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); } else if (multiCols) { char uname[TSDB_COL_NAME_LEN] = {0}; - int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); + int32_t len = TMIN(pToken->n + 1, TSDB_COL_NAME_LEN); tstrncpy(uname, pToken->z, len); if (tsKeepOriginalColumnName) { // keep the original column name @@ -1944,7 +1956,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f tstrncpy(name, tmp, TSDB_COL_NAME_LEN); } } else { // use the user-input result column name - int32_t len = MIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); + int32_t len = TMIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); tstrncpy(name, pItem->pNode->exprToken.z, len); } } @@ -2902,6 +2914,8 @@ int32_t doAddOneProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S } pQueryInfo->info.projectionQuery = true; + + taosArrayDestroy(pColumnList); return TSDB_CODE_SUCCESS; } @@ -2948,7 +2962,7 @@ static SSchema createConstantColumnSchema(SVariant* pVal, const SToken* exprStr, if (name != NULL) { tstrncpy(s.name, name, sizeof(s.name)); } else { - size_t tlen = MIN(sizeof(s.name), exprStr->n + 1); + size_t tlen = TMIN(sizeof(s.name), exprStr->n + 1); tstrncpy(s.name, exprStr->z, tlen); strdequote(s.name); } @@ -3026,7 +3040,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* SSchema colSchema = createConstantColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName); char token[TSDB_COL_NAME_LEN] = {0}; - tstrncpy(token, pItem->pNode->exprToken.z, MIN(TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN)); + tstrncpy(token, pItem->pNode->exprToken.z, TMIN(TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN)); STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &colSchema); @@ -3983,5 +3997,9 @@ int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtI validateSqlNode(p, pQueryInfo, &buf); } + taosArrayDestroy(data.pTableMeta); + taosArrayDestroy(req.pUdf); + taosArrayDestroy(req.pTableName); + return code; } diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index bc4eae7ae9..ece3d5f5eb 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -249,7 +249,7 @@ static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlo } } -void destroyDataBlock(STableDataBlocks* pDataBlock) { +static void destroyDataBlock(STableDataBlocks* pDataBlock) { if (pDataBlock == NULL) { return; } @@ -273,12 +273,29 @@ void destroyBlockArrayList(SArray* pDataBlockList) { size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { - destroyDataBlock(taosArrayGetP(pDataBlockList, i)); + void* p = taosArrayGetP(pDataBlockList, i); + destroyDataBlock(p); } taosArrayDestroy(pDataBlockList); } +void destroyBlockHashmap(SHashObj* pDataBlockHash) { + if (pDataBlockHash == NULL) { + return; + } + + void** p1 = taosHashIterate(pDataBlockHash, NULL); + while (p1) { + STableDataBlocks* pBlocks = *p1; + destroyDataBlock(pBlocks); + + p1 = taosHashIterate(pDataBlockHash, p1); + } + + taosHashCleanup(pDataBlockHash); +} + // data block is disordered, sort it in ascending order void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; @@ -490,7 +507,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t } // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; + int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 8b1506bad1..cb7df824b0 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -156,7 +156,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; dst->size = src->size; - SWAP(dst->pData, src->pData, char*); + TSWAP(dst->pData, src->pData, char*); buildMsgHeader(dst); taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); } @@ -525,10 +525,28 @@ static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { tdDestroyKVRowBuilder(&pCxt->tagsBuilder); } +static void destroyDataBlock(STableDataBlocks* pDataBlock) { + if (pDataBlock == NULL) { + return; + } + + tfree(pDataBlock->pData); + if (!pDataBlock->cloned) { + // free the refcount for metermeta + if (pDataBlock->pTableMeta != NULL) { + tfree(pDataBlock->pTableMeta); + } + + destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + } + tfree(pDataBlock); +} + static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyInsertParseContextForTable(pCxt); taosHashCleanup(pCxt->pVgroupsHashObj); - taosHashCleanup(pCxt->pTableBlockHashObj); + + destroyBlockHashmap(pCxt->pTableBlockHashObj); destroyBlockArrayList(pCxt->pTableDataBlocks); destroyBlockArrayList(pCxt->pVgDataBlocks); } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 4271aae451..c98f787f0b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -248,10 +248,15 @@ void qDestroyQuery(SQueryNode* pQueryNode) { if (NULL == pQueryNode) { return; } - if (nodeType(pQueryNode) == TSDB_SQL_INSERT || nodeType(pQueryNode) == TSDB_SQL_CREATE_TABLE) { + + int32_t type = nodeType(pQueryNode); + if (type == TSDB_SQL_INSERT || type == TSDB_SQL_CREATE_TABLE) { SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode; taosArrayDestroy(pModifInfo->pDataBlocks); - } - tfree(pQueryNode); + tfree(pQueryNode); + } else if (type == TSDB_SQL_SELECT) { + SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo*) pQueryNode; + destroyQueryInfo(pQueryStmtInfo); + } } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index b8545b7486..a87e138ed0 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -732,18 +732,8 @@ void cleanupFieldInfo(SFieldInfo* pFieldInfo) { return; } - if (pFieldInfo->internalField != NULL) { - size_t num = taosArrayGetSize(pFieldInfo->internalField); - for (int32_t i = 0; i < num; ++i) { -// SInternalField* pfield = taosArrayGet(pFieldInfo->internalField, i); -// if (pfield->pExpr != NULL && pfield->pExpr->pExpr != NULL) { -// sqlExprDestroy(pfield->pExpr); -// } - } - } - taosArrayDestroy(pFieldInfo->internalField); -// tfree(pFieldInfo->final); + tfree(pFieldInfo->final); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 1aa8836cb2..9a2ca2da98 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -191,10 +191,12 @@ void destroyExprInfo(SExprInfo* pExprInfo) { for(int32_t i = 0; i < pExprInfo->base.numOfParams; ++i) { taosVariantDestroy(&pExprInfo->base.param[i]); } + + tfree(pExprInfo->base.pColumns); tfree(pExprInfo); } -static void dropOneLevelExprInfo(SArray* pExprInfo) { +void dropOneLevelExprInfo(SArray* pExprInfo) { size_t size = taosArrayGetSize(pExprInfo); for (int32_t i = 0; i < size; ++i) { @@ -239,6 +241,9 @@ void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { #endif dst->pExpr = exprdup(src->pExpr); + dst->base.pColumns = calloc(src->base.numOfCols, sizeof(SColumn)); + memcpy(dst->base.pColumns, src->base.pColumns, sizeof(SColumn) * src->base.numOfCols); + memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); for (int32_t j = 0; j < src->base.numOfParams; ++j) { taosVariantAssign(&dst->base.param[j], &src->base.param[j]); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index a6537998f7..6f63feb3c4 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -2232,7 +2232,7 @@ static void yy_reduce( { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} break; case 3: /* cmd ::= SHOW FUNCTIONS */ -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} +{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);} break; case 4: /* cmd ::= SHOW MNODES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index e817b764d5..620dd38360 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -265,7 +265,6 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(const SQueryStmtInfo* } else { // here we can push down the projection to tablescan operator. pNode->numOfExpr = num; - pNode->pExpr = taosArrayInit(num, POINTER_BYTES); taosArrayAddAll(pNode->pExpr, p); } } @@ -357,7 +356,6 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { SArray* exprList = taosArrayInit(4, POINTER_BYTES); if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; -// dropAllExprInfo(exprList); exit(-1); } @@ -373,7 +371,6 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { // 4. add the projection query node SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, &info, exprList, tableColumnList); columnListDestroy(tableColumnList); -// dropAllExprInfo(exprList); taosArrayPush(pDownstream, &pNode); } @@ -398,7 +395,8 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { } static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { - if (pQueryNode->info.type == QNODE_MODIFY) { + int32_t type = nodeType(pQueryNode); + if (type == QNODE_MODIFY) { SDataPayloadInfo* pInfo = pQueryNode->pExtInfo; size_t size = taosArrayGetSize(pInfo->payload); @@ -410,10 +408,17 @@ static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { taosArrayDestroy(pInfo->payload); } + if (type == QNODE_STREAMSCAN || type == QNODE_TABLESCAN) { + SQueryTableInfo* pQueryTableInfo = pQueryNode->pExtInfo; + tfree(pQueryTableInfo->tableName); + } + + printf("----------->Free:%p\n", pQueryNode->pExpr); + taosArrayDestroy(pQueryNode->pExpr); + tfree(pQueryNode->pExtInfo); tfree(pQueryNode->pSchema); tfree(pQueryNode->info.name); -// dropAllExprInfo(pQueryNode->pExpr); if (pQueryNode->pChildren != NULL) { int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 0038c51c7a..12c0d0780b 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -135,7 +135,7 @@ static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks, SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter), pRoot); inserter->numOfTables = pBlocks->numOfTables; inserter->size = pBlocks->size; - SWAP(inserter->pData, pBlocks->pData, char*); + TSWAP(inserter->pData, pBlocks->pData, char*); return (SDataSink*)inserter; } @@ -155,6 +155,16 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si return node; } +static void cleanupPhyNode(SPhyNode* pPhyNode) { + if (pPhyNode == NULL) { + return; + } + + dropOneLevelExprInfo(pPhyNode->pTargets); + tfree(pPhyNode->targetSchema.pSchema); + tfree(pPhyNode); +} + static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { SScanPhyNode* node = (SScanPhyNode*) initPhyNode(pPlanNode, type, size); @@ -445,3 +455,29 @@ void setExchangSourceNode(uint64_t templateId, SDownstreamSource *pSource, SPhyN void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { setExchangSourceNode(templateId, pSource, subplan->pNode); } + +static void destroyDataSinkNode(SDataSink* pSinkNode) { + if (pSinkNode == NULL) { + return; + } + + if (nodeType(pSinkNode) == DSINK_Dispatch) { + SDataDispatcher* pDdSink = (SDataDispatcher*)pSinkNode; + tfree(pDdSink->sink.schema.pSchema); + } + + tfree(pSinkNode); +} + +void qDestroySubplan(SSubplan* pSubplan) { + if (pSubplan == NULL) { + return; + } + + taosArrayDestroy(pSubplan->pChildren); + taosArrayDestroy(pSubplan->pParents); + destroyDataSinkNode(pSubplan->pDataSink); + cleanupPhyNode(pSubplan->pNode); + + tfree(pSubplan); +} diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index dac6f3f825..7f4cec6bc1 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -1123,8 +1123,10 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); -// printf("====Physical plan:====\n"); -// printf("%s\n", *str); + cJSON_Delete(json); + + printf("====Physical plan:====\n"); + printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index c93569a6c1..d546925c5f 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -18,25 +18,6 @@ static void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, int32_t* numOfCols); -static void destroyDataSinkNode(SDataSink* pSinkNode) { - if (pSinkNode == NULL) { - return; - } - tfree(pSinkNode); -} - -void qDestroySubplan(SSubplan* pSubplan) { - if (pSubplan == NULL) { - return; - } - - taosArrayDestroy(pSubplan->pChildren); - taosArrayDestroy(pSubplan->pParents); - destroyDataSinkNode(pSubplan->pDataSink); - // todo destroy pNode - tfree(pSubplan); -} - void qDestroyQueryDag(struct SQueryDag* pDag) { if (pDag == NULL) { return; @@ -51,6 +32,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { SSubplan* pSubplan = taosArrayGetP(pa, j); qDestroySubplan(pSubplan); } + taosArrayDestroy(pa); } @@ -91,16 +73,18 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, return TSDB_CODE_SUCCESS; } -void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, - int32_t* numOfCols) { // extract the final result schema +// extract the final result schema +void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, int32_t* numOfCols) { SArray* pTopSubplan = taosArrayGetP((*pDag)->pSubplans, 0); - SSubplan* pPlan = taosArrayGetP(pTopSubplan, 0); + SSubplan* pPlan = taosArrayGetP(pTopSubplan, 0); SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); *numOfCols = pDataBlockSchema->numOfCols; - *pResSchema = calloc(pDataBlockSchema->numOfCols, sizeof(SSchema)); - memcpy((*pResSchema), pDataBlockSchema->pSchema, pDataBlockSchema->numOfCols * sizeof(SSchema)); + if (*numOfCols > 0) { + *pResSchema = calloc(pDataBlockSchema->numOfCols, sizeof(SSchema)); + memcpy((*pResSchema), pDataBlockSchema->pSchema, pDataBlockSchema->numOfCols * sizeof(SSchema)); + } } void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 90a4388485..cfe2ed96ad 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -85,7 +85,7 @@ static void* pTaskQueue = NULL; int32_t initTaskQueue() { double factor = 4.0; - int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); int32_t queueSize = tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 01c079e1f7..83c7a4208e 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -22,17 +22,17 @@ extern "C" { #include "tlockfree.h" -#define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000 -#define QWORKER_DEFAULT_TASK_NUMBER 10000 -#define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000 - +#define QW_DEFAULT_SCHEDULER_NUMBER 10000 +#define QW_DEFAULT_TASK_NUMBER 10000 +#define QW_DEFAULT_SCH_TASK_NUMBER 10000 +#define QW_DEFAULT_SHORT_RUN_TIMES 2 enum { QW_PHASE_PRE_QUERY = 1, QW_PHASE_POST_QUERY, - QW_PHASE_PRE_CQUERY, - QW_PHASE_POST_CQUERY, QW_PHASE_PRE_FETCH, QW_PHASE_POST_FETCH, + QW_PHASE_PRE_CQUERY, + QW_PHASE_POST_CQUERY, }; enum { @@ -83,11 +83,9 @@ typedef struct SQWMsg { } SQWMsg; typedef struct SQWPhaseInput { - int8_t status; + int8_t taskStatus; int8_t taskType; int32_t code; - qTaskInfo_t taskHandle; - DataSinkHandle sinkHandle; } SQWPhaseInput; typedef struct SQWPhaseOutput { @@ -111,6 +109,7 @@ typedef struct SQWTaskCtx { void *cancelConnection; bool emptyRes; + bool multiExec; int8_t queryContinue; int8_t queryInQueue; int32_t rspCode; @@ -133,7 +132,7 @@ typedef struct SQWorkerMgmt { int8_t nodeType; int32_t nodeId; SRWLatch schLock; - SRWLatch ctxLock; + //SRWLatch ctxLock; SHashObj *schHash; //key: schedulerId, value: SQWSchStatus SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx void *nodeObj; @@ -144,6 +143,8 @@ typedef struct SQWorkerMgmt { #define QW_IDS() sId, qId, tId #define QW_FPARAMS() mgmt, QW_IDS() +#define QW_GET_EVENT_VALUE(ctx, event) atomic_load_8(&(ctx)->events[event]) + #define QW_IS_EVENT_RECEIVED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_RECEIVED) #define QW_IS_EVENT_PROCESSED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_PROCESSED) #define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED) @@ -151,9 +152,10 @@ typedef struct SQWorkerMgmt { #define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) -#define QW_SET_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) +#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) +#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) -#define QW_IN_EXECUTOR(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_FETCH) +#define QW_IS_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) #define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) #define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code)) diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index 7ecc2b2b20..7735e1a1ee 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -30,6 +30,7 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); int32_t qwBuildAndSendDropRsp(void *connection, int32_t code); +int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code); int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len); int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 07f1b0f858..b53fd9ff37 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -11,7 +11,7 @@ SQWDebug gQWDebug = {0}; -int32_t qwValidateStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t oriStatus, int8_t newStatus) { +int32_t qwValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus) { int32_t code = 0; if (oriStatus == newStatus) { @@ -35,6 +35,7 @@ int32_t qwValidateStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ break; case JOB_TASK_STATUS_EXECUTING: if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED + && newStatus != JOB_TASK_STATUS_SUCCEED && newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_CANCELLING && newStatus != JOB_TASK_STATUS_CANCELLED @@ -52,6 +53,12 @@ int32_t qwValidateStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ break; case JOB_TASK_STATUS_SUCCEED: + if (newStatus != JOB_TASK_STATUS_CANCELLED + && newStatus != JOB_TASK_STATUS_DROPPING) { + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; case JOB_TASK_STATUS_FAILED: case JOB_TASK_STATUS_CANCELLING: if (newStatus != JOB_TASK_STATUS_CANCELLED) { @@ -77,7 +84,7 @@ _return: QW_RET(code); } -int32_t qwSetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskStatus *task, int8_t status) { +int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status) { int32_t code = 0; int8_t origStatus = 0; @@ -99,7 +106,7 @@ int32_t qwSetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t } -int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAddSchedulerImpl(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus **sch) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { @@ -125,7 +132,7 @@ int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 return TSDB_CODE_SUCCESS; } -int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { +int32_t qwAcquireSchedulerImpl(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { while (true) { QW_LOCK(rwType, &mgmt->schLock); *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); @@ -152,11 +159,11 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, u return TSDB_CODE_SUCCESS; } -int32_t qwAcquireAddScheduler(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAcquireAddScheduler(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(QW_FPARAMS(), rwType, sch, QW_NOT_EXIST_ADD); } -int32_t qwAcquireScheduler(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAcquireScheduler(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus **sch) { return qwAcquireSchedulerImpl(QW_FPARAMS(), rwType, sch, QW_NOT_EXIST_RET_ERR); } @@ -165,7 +172,7 @@ void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { } -int32_t qwAcquireTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) { +int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); @@ -181,7 +188,7 @@ int32_t qwAcquireTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint -int32_t qwAddTaskStatusImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWSchStatus *sch, int32_t rwType, int32_t status, SQWTaskStatus **task) { +int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, int32_t status, SQWTaskStatus **task) { int32_t code = 0; char id[sizeof(qId) + sizeof(tId)] = {0}; @@ -215,7 +222,7 @@ int32_t qwAddTaskStatusImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status) { +int32_t qwAddTaskStatus(QW_FPARAMS_DEF, int32_t status) { SQWSchStatus *tsch = NULL; int32_t code = 0; QW_ERR_RET(qwAcquireAddScheduler(QW_FPARAMS(), QW_READ, &tsch)); @@ -230,7 +237,7 @@ _return: } -int32_t qwAddAcquireTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWSchStatus *sch, int32_t status, SQWTaskStatus **task) { +int32_t qwAddAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, int32_t status, SQWTaskStatus **task) { return qwAddTaskStatusImpl(QW_FPARAMS(), sch, rwType, status, task); } @@ -240,48 +247,48 @@ void qwReleaseTaskStatus(int32_t rwType, SQWSchStatus *sch) { } -int32_t qwAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWTaskCtx **ctx) { +int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - QW_LOCK(rwType, &mgmt->ctxLock); - *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); + //QW_LOCK(rwType, &mgmt->ctxLock); + *ctx = taosHashAcquire(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { - QW_UNLOCK(rwType, &mgmt->ctxLock); - QW_TASK_ELOG("ctx not in ctxHash, id:%s", id); - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + //QW_UNLOCK(rwType, &mgmt->ctxLock); + QW_TASK_DLOG_E("task ctx not exist, may be dropped"); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } return TSDB_CODE_SUCCESS; } -int32_t qwGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { +int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { - QW_TASK_ELOG("ctx not in ctxHash, ctxHashSize:%d", taosHashGetSize(mgmt->ctxHash)); - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + QW_TASK_DLOG_E("task ctx not exist, may be dropped"); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, int32_t status, SQWTaskCtx **ctx) { +int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, int32_t status, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); SQWTaskCtx nctx = {0}; - QW_LOCK(QW_WRITE, &mgmt->ctxLock); + //QW_LOCK(QW_WRITE, &mgmt->ctxLock); int32_t code = taosHashPut(mgmt->ctxHash, id, sizeof(id), &nctx, sizeof(SQWTaskCtx)); if (0 != code) { - QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + //QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); if (HASH_NODE_EXIST(code)) { - if (rwType && ctx) { - QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); + if (acquire && ctx) { + QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), ctx)); } else if (ctx) { QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); } else { @@ -293,10 +300,10 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } - QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + //QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); - if (rwType && ctx) { - QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), rwType, ctx)); + if (acquire && ctx) { + QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), ctx)); } else if (ctx) { QW_RET(qwGetTaskCtx(QW_FPARAMS(), ctx)); } @@ -304,23 +311,24 @@ int32_t qwAddTaskCtxImpl(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { - QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), 0, 0, NULL)); +int32_t qwAddTaskCtx(QW_FPARAMS_DEF) { + QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), false, 0, NULL)); } -int32_t qwAddAcquireTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t rwType, SQWTaskCtx **ctx) { - return qwAddTaskCtxImpl(QW_FPARAMS(), rwType, 0, ctx); +int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { + return qwAddTaskCtxImpl(QW_FPARAMS(), true, 0, ctx); } -int32_t qwAddGetTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWTaskCtx **ctx) { - return qwAddTaskCtxImpl(QW_FPARAMS(), 0, 0, ctx); +int32_t qwAddGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { + return qwAddTaskCtxImpl(QW_FPARAMS(), false, 0, ctx); } -void qwReleaseTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt) { - QW_UNLOCK(rwType, &mgmt->ctxLock); +void qwReleaseTaskCtx(SQWorkerMgmt *mgmt, void *ctx) { + //QW_UNLOCK(rwType, &mgmt->ctxLock); + taosHashRelease(mgmt->ctxHash, ctx); } void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { @@ -355,21 +363,30 @@ void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { // Note: NEED CTX HASH LOCKED BEFORE ENTRANCE -int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { +int32_t qwDropTaskCtx(QW_FPARAMS_DEF, int32_t rwType) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); SQWTaskCtx octx; SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == ctx) { - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } octx = *ctx; + atomic_store_ptr(&ctx->taskHandle, NULL); + atomic_store_ptr(&ctx->sinkHandle, NULL); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_DROP); + + if (rwType) { + QW_UNLOCK(rwType, &ctx->lock); + } + if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { QW_TASK_ELOG_E("taosHashRemove from ctx hash failed"); - QW_ERR_RET(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } if (octx.taskHandle) { @@ -386,7 +403,7 @@ int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t } -int32_t qwDropTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { +int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -421,7 +438,7 @@ _return: QW_RET(code); } -int32_t qwUpdateTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status) { +int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -439,12 +456,15 @@ _return: QW_RET(code); } -int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkHandle, int8_t taskType) { +int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t code = 0; bool qcontinue = true; SSDataBlock* pRes = NULL; uint64_t useconds = 0; int32_t i = 0; + int32_t execNum = 0; + qTaskInfo_t *taskHandle = &ctx->taskHandle; + DataSinkHandle sinkHandle = ctx->sinkHandle; while (true) { QW_TASK_DLOG("start to execTask in executor, loopIdx:%d", i++); @@ -455,14 +475,17 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkH QW_ERR_JRET(code); } + ++execNum; + if (NULL == pRes) { QW_TASK_DLOG("task query done, useconds:%"PRIu64, useconds); dsEndPut(sinkHandle, useconds); - if (TASK_TYPE_TEMP == taskType) { + if (TASK_TYPE_TEMP == ctx->taskType) { qwFreeTaskHandle(QW_FPARAMS(), taskHandle); } + break; } @@ -478,6 +501,14 @@ int32_t qwExecTask(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle, DataSinkHandle sinkH if (!qcontinue) { break; } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY) && execNum >= QW_DEFAULT_SHORT_RUN_TIMES) { + break; + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + break; + } } _return: @@ -531,6 +562,8 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void return TSDB_CODE_SUCCESS; } + + pOutput->bufStatus = DS_BUF_EMPTY; QW_TASK_DLOG("no res data in sink, need response later, queryEnd:%d", queryEnd); @@ -567,28 +600,33 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void return TSDB_CODE_SUCCESS; } - -int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { +int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; int8_t status = 0; SQWTaskCtx *ctx = NULL; bool locked = false; - bool ctxAcquired = false; - void *readyConnection = NULL; void *dropConnection = NULL; void *cancelConnection = NULL; QW_SCH_TASK_DLOG("start to handle event at phase %d", phase); + output->needStop = false; + + if (QW_PHASE_PRE_QUERY == phase) { + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); + } else { + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + } + + QW_LOCK(QW_WRITE, &ctx->lock); + locked = true; + switch (phase) { case QW_PHASE_PRE_QUERY: { - QW_ERR_JRET(qwAddGetTaskCtx(QW_FPARAMS(), &ctx)); - - atomic_store_32(&ctx->phase, phase); - atomic_store_8(&ctx->taskType, input->taskType); + atomic_store_8(&ctx->phase, phase); - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_ELOG("task already cancelled at wrong phase, phase:%d", phase); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_ELOG("task already cancelled/dropped at wrong phase, phase:%d", phase); output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; @@ -597,22 +635,35 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_SET_RSP_CODE(ctx, output->rspCode); dropConnection = ctx->dropConnection; // Note: ctx freed, no need to unlock it - locked = false; + locked = false; + + break; } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - output->needStop = true; - QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - + output->needStop = true; output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_SET_RSP_CODE(ctx, output->rspCode); + + cancelConnection = ctx->cancelConnection; + + break; + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed at wrong phase, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); } if (!output->needStop) { @@ -620,65 +671,13 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ } break; } - case QW_PHASE_POST_QUERY: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - ctx->taskHandle = input->taskHandle; - ctx->sinkHandle = input->sinkHandle; - - if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { - ctx->emptyRes = true; - } - - if (input->code) { - output->rspCode = input->code; - } - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - output->needStop = true; - - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); - - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; - dropConnection = ctx->dropConnection; - - // Note: ctx freed, no need to unlock it - locked = false; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - output->needStop = true; - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - qwFreeTask(QW_FPARAMS(), ctx); - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) { - readyConnection = ctx->readyConnection; - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); - } - - if (!output->needStop) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), input->status)); - } - break; - } case QW_PHASE_PRE_FETCH: { - QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); - ctxAcquired = true; - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - atomic_store_32(&ctx->phase, phase); - + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already dropped, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + } if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { QW_TASK_WLOG("task already cancelled, phase:%d", phase); output->needStop = true; @@ -698,6 +697,13 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } + if (ctx->rspCode) { + QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { QW_TASK_WLOG("last fetch not finished, phase:%d", phase); output->needStop = true; @@ -711,74 +717,10 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ output->rspCode = TSDB_CODE_QRY_TASK_MSG_ERROR; QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); } - - if (ctx->rspCode) { - QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } - break; - } - case QW_PHASE_POST_FETCH: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - if (input->code) { - output->rspCode = input->code; - } - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("start to drop task, phase:%d", phase); - output->needStop = true; - - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); - - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; - dropConnection = ctx->dropConnection; - - // Note: ctx freed, no need to unlock it - locked = false; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("start to cancel task, phase:%d", phase); - output->needStop = true; - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - cancelConnection = ctx->cancelConnection; - } - - if (ctx->rspCode) { - QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } break; } case QW_PHASE_PRE_CQUERY: { - QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), QW_READ, &ctx)); - ctxAcquired = true; - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - atomic_store_32(&ctx->phase, phase); + atomic_store_8(&ctx->phase, phase); if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { QW_TASK_WLOG("task already cancelled, phase:%d", phase); @@ -787,18 +729,41 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); } - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_ELOG("drop event at wrong phase, phase:%d", phase); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already dropped, phase:%d", phase); output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_ELOG("cancel event at wrong phase, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_STATUS_ERROR; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); + + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + output->needStop = true; + QW_SET_RSP_CODE(ctx, output->rspCode); + dropConnection = ctx->dropConnection; + + // Note: ctx freed, no need to unlock it + locked = false; + + QW_ERR_JRET(output->rspCode); + } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); + qwFreeTask(QW_FPARAMS(), ctx); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); + + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_SET_RSP_CODE(ctx, output->rspCode); + cancelConnection = ctx->cancelConnection; + + QW_ERR_JRET(output->rspCode); + } + + if (ctx->rspCode) { QW_TASK_ELOG("task already failed, code:%x, phase:%d", ctx->rspCode, phase); output->needStop = true; @@ -807,72 +772,153 @@ int32_t qwHandleTaskEvent(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQ } break; } - case QW_PHASE_POST_CQUERY: { - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - - QW_LOCK(QW_WRITE, &ctx->lock); - - locked = true; - - if (input->code) { - output->rspCode = input->code; - } - - if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("task already cancelled, phase:%d", phase); - output->needStop = true; - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); - } - - if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_TASK_WLOG("start to drop task, phase:%d", phase); - output->needStop = true; - - QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); - - output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; - dropConnection = ctx->dropConnection; - - // Note: ctx freed, no need to unlock it - locked = false; - } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { - QW_TASK_WLOG("start to cancel task, phase:%d", phase); - output->needStop = true; - - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); - - output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; - cancelConnection = ctx->cancelConnection; - } - - if (ctx->rspCode) { - QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); - output->needStop = true; - output->rspCode = ctx->rspCode; - QW_ERR_JRET(output->rspCode); - } - break; - } } _return: - if (output->rspCode) { - QW_SET_RSP_CODE(ctx, output->rspCode); - } - - if (locked) { - atomic_store_32(&ctx->phase, phase); + if (ctx) { + if (output->rspCode) { + QW_UPDATE_RSP_CODE(ctx, output->rspCode); + } - QW_UNLOCK(QW_WRITE, &ctx->lock); + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + + qwReleaseTaskCtx(mgmt, ctx); } - if (ctxAcquired && ctx) { - qwReleaseTaskCtx(QW_READ, mgmt); + if (code) { + output->needStop = true; + if (TSDB_CODE_SUCCESS == output->rspCode) { + output->rspCode = code; + } + } + + if (dropConnection) { + qwBuildAndSendDropRsp(dropConnection, output->rspCode); + QW_TASK_DLOG("drop msg rsped, code:%x", output->rspCode); + } + + if (cancelConnection) { + qwBuildAndSendCancelRsp(cancelConnection, output->rspCode); + QW_TASK_DLOG("cancel msg rsped, code:%x", output->rspCode); + } + + QW_SCH_TASK_DLOG("end to handle event at phase %d", phase); + + QW_RET(code); +} + + +int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { + int32_t code = 0; + int8_t status = 0; + SQWTaskCtx *ctx = NULL; + bool locked = false; + void *readyConnection = NULL; + void *dropConnection = NULL; + void *cancelConnection = NULL; + + QW_SCH_TASK_DLOG("start to handle event at phase %d", phase); + + output->needStop = false; + + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + locked = true; + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already dropped, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); + } + + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL)) { + QW_TASK_WLOG("task already cancelled, phase:%d", phase); + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + if (input->code) { + output->rspCode = input->code; + } + + if (QW_PHASE_POST_QUERY == phase) { + if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { + ctx->emptyRes = true; + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) { + readyConnection = ctx->readyConnection; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); + } + } + + if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); + + output->rspCode = TSDB_CODE_QRY_TASK_DROPPED; + output->needStop = true; + QW_SET_RSP_CODE(ctx, output->rspCode); + dropConnection = ctx->dropConnection; + + // Note: ctx freed, no need to unlock it + locked = false; + + QW_ERR_JRET(output->rspCode); + } else if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL)) { + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); + qwFreeTask(QW_FPARAMS(), ctx); + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL); + + output->needStop = true; + output->rspCode = TSDB_CODE_QRY_TASK_CANCELLED; + QW_SET_RSP_CODE(ctx, output->rspCode); + cancelConnection = ctx->cancelConnection; + + QW_ERR_JRET(output->rspCode); + } + + if (ctx->rspCode) { + QW_TASK_ELOG("task failed, code:%x, phase:%d", ctx->rspCode, phase); + output->needStop = true; + output->rspCode = ctx->rspCode; + QW_ERR_JRET(output->rspCode); + } + + if (QW_PHASE_POST_QUERY == phase && (!output->needStop)) { + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), input->taskStatus)); + } + +_return: + + if (ctx) { + if (output->rspCode) { + QW_UPDATE_RSP_CODE(ctx, output->rspCode); + } + + if (QW_PHASE_POST_FETCH != phase) { + atomic_store_8(&ctx->phase, phase); + } + + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + + qwReleaseTaskCtx(mgmt, ctx); + } + + if (code) { + output->needStop = true; + if (TSDB_CODE_SUCCESS == output->rspCode) { + output->rspCode = code; + } } if (readyConnection) { @@ -896,7 +942,7 @@ _return: } -int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg, int8_t taskType) { +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { int32_t code = 0; bool queryRsped = false; bool needStop = false; @@ -906,10 +952,9 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t SQWPhaseOutput output = {0}; qTaskInfo_t pTaskInfo = NULL; DataSinkHandle sinkHandle = NULL; + SQWTaskCtx *ctx = NULL; - input.taskType = taskType; - - QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, &output)); + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, &output)); needStop = output.needStop; code = output.rspCode; @@ -918,6 +963,10 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_TASK_DLOG("task need stop, phase:%d", QW_PHASE_PRE_QUERY); QW_ERR_JRET(code); } + + QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + + atomic_store_8(&ctx->taskType, taskType); code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { @@ -943,8 +992,11 @@ int32_t qwProcessQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t queryRsped = true; + atomic_store_ptr(&ctx->taskHandle, pTaskInfo); + atomic_store_ptr(&ctx->sinkHandle, sinkHandle); + if (pTaskInfo && sinkHandle) { - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &pTaskInfo, sinkHandle, taskType)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx)); } _return: @@ -954,46 +1006,35 @@ _return: } if (!queryRsped) { - code = qwBuildAndSendQueryRsp(qwMsg->connection, rspCode); - if (TSDB_CODE_SUCCESS == code) { - QW_TASK_DLOG("query msg rsped, code:%d", rspCode); - } - - if (TSDB_CODE_SUCCESS == rspCode && code) { - rspCode = code; - } - } - - if (needStop) { - QW_RET(rspCode); + qwBuildAndSendQueryRsp(qwMsg->connection, rspCode); + QW_TASK_DLOG("query msg rsped, code:%x", rspCode); } input.code = rspCode; - input.taskHandle = pTaskInfo; - input.sinkHandle = sinkHandle; + input.taskStatus = rspCode ? JOB_TASK_STATUS_FAILED : JOB_TASK_STATUS_PARTIAL_SUCCEED; - if (TSDB_CODE_SUCCESS != rspCode) { - input.status = JOB_TASK_STATUS_FAILED; - } else { - input.status = JOB_TASK_STATUS_PARTIAL_SUCCEED; - } - - QW_ERR_RET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); + QW_ERR_RET(qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, &output)); QW_RET(rspCode); } -int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; SQWTaskCtx *ctx = NULL; int8_t phase = 0; bool needRsp = false; int32_t rspCode = 0; - - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); + + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); + if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP) || + QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_CANCEL) || QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task already cancelled/dropped, phase:%d", phase); + QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + } + phase = QW_GET_PHASE(ctx); if (phase == QW_PHASE_PRE_QUERY) { @@ -1015,11 +1056,12 @@ int32_t qwProcessReady(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t _return: if (code && ctx) { - QW_SET_RSP_CODE(ctx, code); + QW_UPDATE_RSP_CODE(ctx, code); } if (ctx) { QW_UNLOCK(QW_WRITE, &ctx->lock); + qwReleaseTaskCtx(mgmt, ctx); } if (needRsp) { @@ -1031,7 +1073,7 @@ _return: } -int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { SQWTaskCtx *ctx = NULL; int32_t code = 0; bool queryRsped = false; @@ -1043,7 +1085,7 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t int32_t dataLen = 0; do { - QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); needStop = output.needStop; code = output.rspCode; @@ -1056,10 +1098,11 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); atomic_store_8(&ctx->queryInQueue, 0); + atomic_store_8(&ctx->queryContinue, 0); DataSinkHandle sinkHandle = ctx->sinkHandle; - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), &ctx->taskHandle, sinkHandle, ctx->taskType)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx)); if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; @@ -1071,6 +1114,10 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t // RC WARNING atomic_store_8(&ctx->queryContinue, 1); } + + if (sOutput.queryEnd) { + needStop = true; + } if (rsp) { qwBuildFetchRsp(rsp, &sOutput, dataLen); @@ -1086,6 +1133,10 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t _return: + if (NULL == ctx) { + break; + } + if (code && QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwFreeFetchRsp(rsp); @@ -1094,18 +1145,24 @@ int32_t qwProcessCQuery(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, 0); } - input.code = code; - qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); + QW_LOCK(QW_WRITE, &ctx->lock); + if (needStop || code || 0 == atomic_load_8(&ctx->queryContinue)) { + atomic_store_8(&ctx->phase, 0); + QW_UNLOCK(QW_WRITE,&ctx->lock); + break; + } + + QW_UNLOCK(QW_WRITE,&ctx->lock); + } while (true); - needStop = output.needStop; - code = output.rspCode; - } while ((!needStop) && (0 == code) && atomic_val_compare_exchange_8(&ctx->queryContinue, 1, 0)); + input.code = code; + qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, &output); QW_RET(code); } -int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; int32_t needRsp = true; void *data = NULL; @@ -1121,7 +1178,7 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t SQWPhaseInput input = {0}; SQWPhaseOutput output = {0}; - QW_ERR_JRET(qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, &output)); + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, &output)); needStop = output.needStop; code = output.rspCode; @@ -1149,14 +1206,19 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t locked = true; // RC WARNING - if (QW_IN_EXECUTOR(ctx)) { + if (QW_IS_QUERY_RUNNING(ctx)) { atomic_store_8(&ctx->queryContinue, 1); } else if (0 == atomic_load_8(&ctx->queryInQueue)) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); + if (!ctx->multiExec) { + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); + ctx->multiExec = true; + } atomic_store_8(&ctx->queryInQueue, 1); QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); + + QW_TASK_DLOG("schedule query in queue, phase:%d", ctx->phase); } } @@ -1168,7 +1230,11 @@ _return: input.code = code; - qwHandleTaskEvent(QW_FPARAMS(), QW_PHASE_POST_FETCH, &input, &output); + qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_FETCH, &input, &output); + + if (output.rspCode) { + code = output.rspCode; + } if (code) { qwFreeFetchRsp(rsp); @@ -1185,13 +1251,13 @@ _return: } -int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg) { +int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; bool needRsp = false; SQWTaskCtx *ctx = NULL; bool locked = false; - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), QW_WRITE, &ctx)); + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -1202,28 +1268,30 @@ int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t QW_ERR_JRET(TSDB_CODE_QRY_DUPLICATTED_OPERATION); } - if (QW_IN_EXECUTOR(ctx)) { + if (QW_IS_QUERY_RUNNING(ctx)) { QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING)); - - ctx->dropConnection = qwMsg->connection; } else if (ctx->phase > 0) { QW_ERR_JRET(qwDropTaskStatus(QW_FPARAMS())); - QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS())); + QW_ERR_JRET(qwDropTaskCtx(QW_FPARAMS(), QW_WRITE)); + + QW_SET_RSP_CODE(ctx, TSDB_CODE_QRY_TASK_DROPPED); locked = false; needRsp = true; } - if (!needRsp) { + if (!needRsp) { + ctx->dropConnection = qwMsg->connection; + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } _return: if (code) { - QW_SET_RSP_CODE(ctx, code); + QW_UPDATE_RSP_CODE(ctx, code); } if (locked) { @@ -1231,7 +1299,7 @@ _return: } if (ctx) { - qwReleaseTaskCtx(QW_WRITE, mgmt); + qwReleaseTaskCtx(mgmt, ctx); } if (TSDB_CODE_SUCCESS != code || needRsp) { @@ -1258,18 +1326,18 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW if (cfg) { mgmt->cfg = *cfg; if (0 == mgmt->cfg.maxSchedulerNum) { - mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; + mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; } if (0 == mgmt->cfg.maxTaskNum) { - mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER; + mgmt->cfg.maxTaskNum = QW_DEFAULT_TASK_NUMBER; } if (0 == mgmt->cfg.maxSchTaskNum) { - mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; + mgmt->cfg.maxSchTaskNum = QW_DEFAULT_SCH_TASK_NUMBER; } } else { - mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; - mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER; - mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; + mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; + mgmt->cfg.maxTaskNum = QW_DEFAULT_TASK_NUMBER; + mgmt->cfg.maxSchTaskNum = QW_DEFAULT_SCH_TASK_NUMBER; } mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index feb8fd645e..d11bee6dce 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -50,6 +50,7 @@ int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { + .msgType = TDMT_VND_QUERY_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -68,6 +69,7 @@ int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { + .msgType = TDMT_VND_RES_READY_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -98,7 +100,7 @@ int32_t qwBuildAndSendStatusRsp(SRpcMsg *pMsg, SSchedulerStatusRsp *sStatus) { } SRpcMsg rpcRsp = { - .msgType = pMsg->msgType + 1, + .msgType = TDMT_VND_TASKS_STATUS_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -121,6 +123,7 @@ int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_ } SRpcMsg rpcRsp = { + .msgType = TDMT_VND_FETCH_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -138,6 +141,7 @@ int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { + .msgType = TDMT_VND_CANCEL_TASK_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -155,6 +159,7 @@ int32_t qwBuildAndSendDropRsp(void *connection, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { + .msgType = TDMT_VND_DROP_TASK_RSP, .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, @@ -273,7 +278,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { - QW_ELOG("invalid query msg, contLen:%d", pMsg->contLen); + QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -306,15 +311,11 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWTaskCtx *handles = NULL; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { - QW_ELOG("invalid cquery msg, contLen:%d", pMsg->contLen); + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { + QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); - msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - uint64_t sId = msg->sId; uint64_t qId = msg->queryId; uint64_t tId = msg->taskId; @@ -335,14 +336,13 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ return TSDB_CODE_QRY_INVALID_INPUT; } + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; SResReadyReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { - qError("invalid task status msg"); + QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; - msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); msg->taskId = be64toh(msg->taskId); @@ -398,6 +398,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { + QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 0ed5c0c816..b9d84b25aa 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -44,13 +44,55 @@ namespace { +#define qwtTestQueryQueueSize 1000000 +#define qwtTestFetchQueueSize 1000000 + +int32_t qwtTestMaxExecTaskUsec = 2; +int32_t qwtTestReqMaxDelayUsec = 2; + +uint64_t qwtTestQueryId = 0; bool qwtTestEnableSleep = true; bool qwtTestStop = false; -bool qwtTestDeadLoop = true; -int32_t qwtTestMTRunSec = 10; +bool qwtTestDeadLoop = false; +int32_t qwtTestMTRunSec = 60; int32_t qwtTestPrintNum = 100000; int32_t qwtTestCaseIdx = 0; int32_t qwtTestCaseNum = 4; +bool qwtTestCaseFinished = false; +tsem_t qwtTestQuerySem; +tsem_t qwtTestFetchSem; +int32_t qwtTestQuitThreadNum = 0; + + +int32_t qwtTestQueryQueueRIdx = 0; +int32_t qwtTestQueryQueueWIdx = 0; +int32_t qwtTestQueryQueueNum = 0; +SRWLatch qwtTestQueryQueueLock = 0; +struct SRpcMsg *qwtTestQueryQueue[qwtTestQueryQueueSize] = {0}; + +int32_t qwtTestFetchQueueRIdx = 0; +int32_t qwtTestFetchQueueWIdx = 0; +int32_t qwtTestFetchQueueNum = 0; +SRWLatch qwtTestFetchQueueLock = 0; +struct SRpcMsg *qwtTestFetchQueue[qwtTestFetchQueueSize] = {0}; + + +int32_t qwtTestSinkBlockNum = 0; +int32_t qwtTestSinkMaxBlockNum = 0; +bool qwtTestSinkQueryEnd = false; +SRWLatch qwtTestSinkLock = 0; +int32_t qwtTestSinkLastLen = 0; + + +SSubQueryMsg qwtqueryMsg = {0}; +SRpcMsg qwtfetchRpc = {0}; +SResFetchReq qwtfetchMsg = {0}; +SRpcMsg qwtreadyRpc = {0}; +SResReadyReq qwtreadyMsg = {0}; +SRpcMsg qwtdropRpc = {0}; +STaskDropReq qwtdropMsg = {0}; +SSchTasksStatusReq qwtstatusMsg = {0}; + void qwtInitLogFile() { const char *defaultLogFileNamePrefix = "taosdlog"; @@ -68,35 +110,38 @@ void qwtInitLogFile() { } void qwtBuildQueryReqMsg(SRpcMsg *queryRpc) { - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc->pCont = queryMsg; + qwtqueryMsg.queryId = htobe64(atomic_add_fetch_64(&qwtTestQueryId, 1)); + qwtqueryMsg.sId = htobe64(1); + qwtqueryMsg.taskId = htobe64(1); + qwtqueryMsg.contentLen = htonl(100); + queryRpc->msgType = TDMT_VND_QUERY; + queryRpc->pCont = &qwtqueryMsg; queryRpc->contLen = sizeof(SSubQueryMsg) + 100; } void qwtBuildReadyReqMsg(SResReadyReq *readyMsg, SRpcMsg *readyRpc) { readyMsg->sId = htobe64(1); - readyMsg->queryId = htobe64(1); + readyMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); readyMsg->taskId = htobe64(1); + readyRpc->msgType = TDMT_VND_RES_READY; readyRpc->pCont = readyMsg; readyRpc->contLen = sizeof(SResReadyReq); } void qwtBuildFetchReqMsg(SResFetchReq *fetchMsg, SRpcMsg *fetchRpc) { fetchMsg->sId = htobe64(1); - fetchMsg->queryId = htobe64(1); + fetchMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); fetchMsg->taskId = htobe64(1); + fetchRpc->msgType = TDMT_VND_FETCH; fetchRpc->pCont = fetchMsg; fetchRpc->contLen = sizeof(SResFetchReq); } void qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) { dropMsg->sId = htobe64(1); - dropMsg->queryId = htobe64(1); + dropMsg->queryId = htobe64(atomic_load_64(&qwtTestQueryId)); dropMsg->taskId = htobe64(1); + dropRpc->msgType = TDMT_VND_DROP_TASK; dropRpc->pCont = dropMsg; dropRpc->contLen = sizeof(STaskDropReq); } @@ -109,30 +154,121 @@ void qwtBuildStatusReqMsg(SSchTasksStatusReq *statusMsg, SRpcMsg *statusRpc) { } int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { + *subplan = (SSubplan *)0x1; return 0; } -int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { +int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { + taosWLockLatch(&qwtTestFetchQueueLock); + struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); + qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg; + if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) { + qwtTestFetchQueueWIdx = 0; + } + + qwtTestFetchQueueNum++; + + if (qwtTestFetchQueueWIdx == qwtTestFetchQueueRIdx) { + printf("Fetch queue is full"); + assert(0); + } + taosWUnLockLatch(&qwtTestFetchQueueLock); + + tsem_post(&qwtTestFetchSem); + return 0; } +int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { + taosWLockLatch(&qwtTestQueryQueueLock); + struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); + qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg; + if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { + qwtTestQueryQueueWIdx = 0; + } + + qwtTestQueryQueueNum++; + + if (qwtTestQueryQueueWIdx == qwtTestQueryQueueRIdx) { + printf("query queue is full"); + assert(0); + } + taosWUnLockLatch(&qwtTestQueryQueueLock); + + tsem_post(&qwtTestQuerySem); + + return 0; +} + + + void qwtRpcSendResponse(const SRpcMsg *pRsp) { -/* - if (TDMT_VND_TASKS_STATUS_RSP == pRsp->msgType) { - SSchedulerStatusRsp *rsp = (SSchedulerStatusRsp *)pRsp->pCont; - printf("task num:%d\n", rsp->num); - for (int32_t i = 0; i < rsp->num; ++i) { - STaskStatus *task = &rsp->status[i]; - printf("qId:%"PRIx64",tId:%"PRIx64",status:%d\n", task->queryId, task->taskId, task->status); + + switch (pRsp->msgType) { + case TDMT_VND_QUERY_RSP: { + SQueryTableRsp *rsp = (SQueryTableRsp *)pRsp->pCont; + + if (0 == pRsp->code) { + qwtBuildReadyReqMsg(&qwtreadyMsg, &qwtreadyRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtreadyRpc); + } else { + qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); + } + + rpcFreeCont(rsp); + break; + } + case TDMT_VND_RES_READY_RSP: { + SResReadyRsp *rsp = (SResReadyRsp *)pRsp->pCont; + + if (0 == pRsp->code) { + qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtfetchRpc); + } else { + qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); + } + rpcFreeCont(rsp); + break; + } + case TDMT_VND_FETCH_RSP: { + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)pRsp->pCont; + + if (0 == pRsp->code && 0 == rsp->completed) { + qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtfetchRpc); + return; + } + + qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); + qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); + rpcFreeCont(rsp); + + break; + } + case TDMT_VND_DROP_TASK_RSP: { + STaskDropRsp *rsp = (STaskDropRsp *)pRsp->pCont; + rpcFreeCont(rsp); + + qwtTestCaseFinished = true; + break; } } -*/ + + return; } int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { - int32_t idx = qwtTestCaseIdx % qwtTestCaseNum; + int32_t idx = abs((++qwtTestCaseIdx) % qwtTestCaseNum); + + qwtTestSinkBlockNum = 0; + qwtTestSinkMaxBlockNum = rand() % 100 + 1; + qwtTestSinkQueryEnd = false; if (0 == idx) { *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx; @@ -147,13 +283,45 @@ int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTas *pTaskInfo = NULL; *handle = (DataSinkHandle)qwtTestCaseIdx; } - - ++qwtTestCaseIdx; return 0; } int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { + int32_t endExec = 0; + + if (NULL == tinfo) { + *pRes = NULL; + *useconds = 0; + } else { + if (qwtTestSinkQueryEnd) { + *pRes = NULL; + *useconds = rand() % 10; + return 0; + } + + endExec = rand() % 5; + + int32_t runTime = 0; + if (qwtTestEnableSleep && qwtTestMaxExecTaskUsec > 0) { + runTime = rand() % qwtTestMaxExecTaskUsec; + } + + if (qwtTestEnableSleep) { + if (runTime) { + usleep(runTime); + } + } + + if (endExec) { + *pRes = (SSDataBlock*)calloc(1, sizeof(SSDataBlock)); + (*pRes)->info.rows = rand() % 1000; + } else { + *pRes = NULL; + *useconds = rand() % 10; + } + } + return 0; } @@ -162,21 +330,100 @@ int32_t qwtKillTask(qTaskInfo_t qinfo) { } void qwtDestroyTask(qTaskInfo_t qHandle) { - } int32_t qwtPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue) { + if (NULL == handle || NULL == pInput || NULL == pContinue) { + assert(0); + } + + free((void *)pInput->pData); + + taosWLockLatch(&qwtTestSinkLock); + + qwtTestSinkBlockNum++; + + if (qwtTestSinkBlockNum >= qwtTestSinkMaxBlockNum) { + *pContinue = false; + } else { + *pContinue = true; + } + taosWUnLockLatch(&qwtTestSinkLock); + return 0; } void qwtEndPut(DataSinkHandle handle, uint64_t useconds) { + if (NULL == handle) { + assert(0); + } + + qwtTestSinkQueryEnd = true; } void qwtGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { + static int32_t in = 0; + + if (in > 0) { + assert(0); + } + + atomic_add_fetch_32(&in, 1); + + if (NULL == handle) { + assert(0); + } + + taosWLockLatch(&qwtTestSinkLock); + if (qwtTestSinkBlockNum > 0) { + *pLen = rand() % 100 + 1; + qwtTestSinkBlockNum--; + } else { + *pLen = 0; + } + qwtTestSinkLastLen = *pLen; + taosWUnLockLatch(&qwtTestSinkLock); + + *pQueryEnd = qwtTestSinkQueryEnd; + + atomic_sub_fetch_32(&in, 1); } int32_t qwtGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { + taosWLockLatch(&qwtTestSinkLock); + if (qwtTestSinkLastLen > 0) { + pOutput->numOfRows = rand() % 10 + 1; + pOutput->compressed = 1; + pOutput->queryEnd = qwtTestSinkQueryEnd; + if (qwtTestSinkBlockNum == 0) { + pOutput->bufStatus = DS_BUF_EMPTY; + } else if (qwtTestSinkBlockNum <= qwtTestSinkMaxBlockNum*0.5) { + pOutput->bufStatus = DS_BUF_LOW; + } else { + pOutput->bufStatus = DS_BUF_FULL; + } + pOutput->useconds = rand() % 10 + 1; + pOutput->precision = 1; + } else if (qwtTestSinkLastLen == 0) { + pOutput->numOfRows = 0; + pOutput->compressed = 1; + pOutput->pData = NULL; + pOutput->queryEnd = qwtTestSinkQueryEnd; + if (qwtTestSinkBlockNum == 0) { + pOutput->bufStatus = DS_BUF_EMPTY; + } else if (qwtTestSinkBlockNum <= qwtTestSinkMaxBlockNum*0.5) { + pOutput->bufStatus = DS_BUF_LOW; + } else { + pOutput->bufStatus = DS_BUF_FULL; + } + pOutput->useconds = rand() % 10 + 1; + pOutput->precision = 1; + } else { + assert(0); + } + taosWUnLockLatch(&qwtTestSinkLock); + return 0; } @@ -343,7 +590,6 @@ void *queryThread(void *param) { while (!qwtTestStop) { qwtBuildQueryReqMsg(&queryRpc); qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - free(queryRpc.pCont); if (qwtTestEnableSleep) { usleep(rand()%5); } @@ -444,33 +690,159 @@ void *statusThread(void *param) { } -void *controlThread(void *param) { - SRpcMsg queryRpc = {0}; +void *qwtclientThread(void *param) { int32_t code = 0; uint32_t n = 0; - void *mockPointer = (void *)0x1; void *mgmt = param; + void *mockPointer = (void *)0x1; + SRpcMsg queryRpc = {0}; + + sleep(1); while (!qwtTestStop) { + qwtTestCaseFinished = false; + qwtBuildQueryReqMsg(&queryRpc); - qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - free(queryRpc.pCont); - if (qwtTestEnableSleep) { - usleep(rand()%5); + qwtPutReqToQueue((void *)0x1, &queryRpc); + + while (!qwtTestCaseFinished) { + usleep(1); } + + if (++n % qwtTestPrintNum == 0) { - printf("query:%d\n", n); + printf("case run:%d\n", n); } } + atomic_add_fetch_32(&qwtTestQuitThreadNum, 1); + return NULL; } void *queryQueueThread(void *param) { + void *mockPointer = (void *)0x1; + SRpcMsg *queryRpc = NULL; + void *mgmt = param; + + while (true) { + tsem_wait(&qwtTestQuerySem); + + if (qwtTestStop && qwtTestQueryQueueNum <= 0 && qwtTestCaseFinished) { + break; + } + + taosWLockLatch(&qwtTestQueryQueueLock); + if (qwtTestQueryQueueNum <= 0 || qwtTestQueryQueueRIdx == qwtTestQueryQueueWIdx) { + printf("query queue is empty\n"); + assert(0); + } + + queryRpc = qwtTestQueryQueue[qwtTestQueryQueueRIdx++]; + + if (qwtTestQueryQueueRIdx >= qwtTestQueryQueueSize) { + qwtTestQueryQueueRIdx = 0; + } + + qwtTestQueryQueueNum--; + taosWUnLockLatch(&qwtTestQueryQueueLock); + + + if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) { + int32_t delay = rand() % qwtTestReqMaxDelayUsec; + + if (delay) { + usleep(delay); + } + } + + if (TDMT_VND_QUERY == queryRpc->msgType) { + qWorkerProcessQueryMsg(mockPointer, mgmt, queryRpc); + } else if (TDMT_VND_QUERY_CONTINUE == queryRpc->msgType) { + qWorkerProcessCQueryMsg(mockPointer, mgmt, queryRpc); + } else { + printf("unknown msg in query queue, type:%d\n", queryRpc->msgType); + assert(0); + } + + free(queryRpc); + + if (qwtTestStop && qwtTestQueryQueueNum <= 0 && qwtTestCaseFinished) { + break; + } + } + + atomic_add_fetch_32(&qwtTestQuitThreadNum, 1); + return NULL; } void *fetchQueueThread(void *param) { + void *mockPointer = (void *)0x1; + SRpcMsg *fetchRpc = NULL; + void *mgmt = param; + + while (true) { + tsem_wait(&qwtTestFetchSem); + + if (qwtTestStop && qwtTestFetchQueueNum <= 0 && qwtTestCaseFinished) { + break; + } + + taosWLockLatch(&qwtTestFetchQueueLock); + if (qwtTestFetchQueueNum <= 0 || qwtTestFetchQueueRIdx == qwtTestFetchQueueWIdx) { + printf("Fetch queue is empty\n"); + assert(0); + } + + fetchRpc = qwtTestFetchQueue[qwtTestFetchQueueRIdx++]; + + if (qwtTestFetchQueueRIdx >= qwtTestFetchQueueSize) { + qwtTestFetchQueueRIdx = 0; + } + + qwtTestFetchQueueNum--; + taosWUnLockLatch(&qwtTestFetchQueueLock); + + if (qwtTestEnableSleep && qwtTestReqMaxDelayUsec > 0) { + int32_t delay = rand() % qwtTestReqMaxDelayUsec; + + if (delay) { + usleep(delay); + } + } + + switch (fetchRpc->msgType) { + case TDMT_VND_FETCH: + qWorkerProcessFetchMsg(mockPointer, mgmt, fetchRpc); + break; + case TDMT_VND_RES_READY: + qWorkerProcessReadyMsg(mockPointer, mgmt, fetchRpc); + break; + case TDMT_VND_TASKS_STATUS: + qWorkerProcessStatusMsg(mockPointer, mgmt, fetchRpc); + break; + case TDMT_VND_CANCEL_TASK: + qWorkerProcessCancelMsg(mockPointer, mgmt, fetchRpc); + break; + case TDMT_VND_DROP_TASK: + qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc); + break; + default: + printf("unknown msg type:%d in fetch queue", fetchRpc->msgType); + assert(0); + break; + } + + free(fetchRpc); + + if (qwtTestStop && qwtTestFetchQueueNum <= 0 && qwtTestCaseFinished) { + break; + } + } + + atomic_add_fetch_32(&qwtTestQuitThreadNum, 1); + return NULL; } @@ -478,6 +850,7 @@ void *fetchQueueThread(void *param) { } +#if 0 TEST(seqTest, normalCase) { void *mgmt = NULL; @@ -490,41 +863,11 @@ TEST(seqTest, normalCase) { SRpcMsg statusRpc = {0}; qwtInitLogFile(); - - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc.pCont = queryMsg; - queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - SResReadyReq readyMsg = {0}; - readyMsg.sId = htobe64(1); - readyMsg.queryId = htobe64(1); - readyMsg.taskId = htobe64(1); - readyRpc.pCont = &readyMsg; - readyRpc.contLen = sizeof(SResReadyReq); - - SResFetchReq fetchMsg = {0}; - fetchMsg.sId = htobe64(1); - fetchMsg.queryId = htobe64(1); - fetchMsg.taskId = htobe64(1); - fetchRpc.pCont = &fetchMsg; - fetchRpc.contLen = sizeof(SResFetchReq); - - STaskDropReq dropMsg = {0}; - dropMsg.sId = htobe64(1); - dropMsg.queryId = htobe64(1); - dropMsg.taskId = htobe64(1); - dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropReq); - - SSchTasksStatusReq statusMsg = {0}; - statusMsg.sId = htobe64(1); - statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusReq); - statusRpc.msgType = TDMT_VND_TASKS_STATUS; + qwtBuildQueryReqMsg(&queryRpc); + qwtBuildReadyReqMsg(&qwtreadyMsg, &readyRpc); + qwtBuildFetchReqMsg(&qwtfetchMsg, &fetchRpc); + qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); stubSetStringToPlan(); stubSetRpcSendResponse(); @@ -541,35 +884,19 @@ TEST(seqTest, normalCase) { code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); - code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - ASSERT_EQ(code, 0); - code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); - code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - ASSERT_EQ(code, 0); - code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); - code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - ASSERT_EQ(code, 0); - code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); - code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); - ASSERT_EQ(code, 0); - code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); @@ -586,26 +913,9 @@ TEST(seqTest, cancelFirst) { qwtInitLogFile(); - SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100); - queryMsg->queryId = htobe64(1); - queryMsg->sId = htobe64(1); - queryMsg->taskId = htobe64(1); - queryMsg->contentLen = htonl(100); - queryRpc.pCont = queryMsg; - queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - - STaskDropReq dropMsg = {0}; - dropMsg.sId = htobe64(1); - dropMsg.queryId = htobe64(1); - dropMsg.taskId = htobe64(1); - dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropReq); - - SSchTasksStatusReq statusMsg = {0}; - statusMsg.sId = htobe64(1); - statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusReq); - statusRpc.msgType = TDMT_VND_TASKS_STATUS; + qwtBuildQueryReqMsg(&queryRpc); + qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); stubSetStringToPlan(); stubSetRpcSendResponse(); @@ -613,21 +923,21 @@ TEST(seqTest, cancelFirst) { code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); ASSERT_EQ(code, 0); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - ASSERT_EQ(code, TSDB_CODE_QRY_TASK_DROPPED); + ASSERT_TRUE(0 != code); - statusMsg.sId = htobe64(1); + qwtBuildStatusReqMsg(&qwtstatusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); @@ -668,24 +978,35 @@ TEST(seqTest, randCase) { printf("Query,%d\n", t++); qwtBuildQueryReqMsg(&queryRpc); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); - free(queryRpc.pCont); } else if (r >= maxr/5 && r < maxr * 2/5) { printf("Ready,%d\n", t++); qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); + if (qwtTestEnableSleep) { + usleep(1); + } } else if (r >= maxr * 2/5 && r < maxr* 3/5) { printf("Fetch,%d\n", t++); qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); + if (qwtTestEnableSleep) { + usleep(1); + } } else if (r >= maxr * 3/5 && r < maxr * 4/5) { printf("Drop,%d\n", t++); qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); + if (qwtTestEnableSleep) { + usleep(1); + } } else if (r >= maxr * 4/5 && r < maxr-1) { printf("Status,%d\n", t++); qwtBuildStatusReqMsg(&statusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); + if (qwtTestEnableSleep) { + usleep(1); + } } else { printf("QUIT RAND NOW"); break; @@ -735,7 +1056,257 @@ TEST(seqTest, multithreadRand) { qWorkerDestroy(&mgmt); } -TEST(rcTest, multithread) { +#endif + +TEST(rcTest, shortExecshortDelay) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + + qwtInitLogFile(); + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); + + srand(time(NULL)); + qwtTestStop = false; + qwtTestQuitThreadNum = 0; + + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + ASSERT_EQ(code, 0); + + qwtTestMaxExecTaskUsec = 0; + qwtTestReqMaxDelayUsec = 0; + + tsem_init(&qwtTestQuerySem, 0, 0); + tsem_init(&qwtTestFetchSem, 0, 0); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t t1,t2,t3,t4,t5; + pthread_create(&(t1), &thattr, qwtclientThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, mgmt); + pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + + while (true) { + if (qwtTestQuitThreadNum == 3) { + break; + } + + sleep(1); + + if (qwtTestCaseFinished) { + if (qwtTestQuitThreadNum < 3) { + tsem_post(&qwtTestQuerySem); + tsem_post(&qwtTestFetchSem); + + usleep(10); + } + } + + } + + qwtTestQueryQueueNum = 0; + qwtTestQueryQueueRIdx = 0; + qwtTestQueryQueueWIdx = 0; + qwtTestQueryQueueLock = 0; + qwtTestFetchQueueNum = 0; + qwtTestFetchQueueRIdx = 0; + qwtTestFetchQueueWIdx = 0; + qwtTestFetchQueueLock = 0; + + qWorkerDestroy(&mgmt); +} + +TEST(rcTest, longExecshortDelay) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + + qwtInitLogFile(); + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); + + srand(time(NULL)); + qwtTestStop = false; + qwtTestQuitThreadNum = 0; + + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + ASSERT_EQ(code, 0); + + qwtTestMaxExecTaskUsec = 1000000; + qwtTestReqMaxDelayUsec = 0; + + tsem_init(&qwtTestQuerySem, 0, 0); + tsem_init(&qwtTestFetchSem, 0, 0); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t t1,t2,t3,t4,t5; + pthread_create(&(t1), &thattr, qwtclientThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, mgmt); + pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + + + while (true) { + if (qwtTestQuitThreadNum == 3) { + break; + } + + sleep(1); + + if (qwtTestCaseFinished) { + if (qwtTestQuitThreadNum < 3) { + tsem_post(&qwtTestQuerySem); + tsem_post(&qwtTestFetchSem); + + usleep(10); + } + } + + } + + qwtTestQueryQueueNum = 0; + qwtTestQueryQueueRIdx = 0; + qwtTestQueryQueueWIdx = 0; + qwtTestQueryQueueLock = 0; + qwtTestFetchQueueNum = 0; + qwtTestFetchQueueRIdx = 0; + qwtTestFetchQueueWIdx = 0; + qwtTestFetchQueueLock = 0; + + qWorkerDestroy(&mgmt); +} + + +TEST(rcTest, shortExeclongDelay) { + void *mgmt = NULL; + int32_t code = 0; + void *mockPointer = (void *)0x1; + + qwtInitLogFile(); + + stubSetStringToPlan(); + stubSetRpcSendResponse(); + stubSetExecTask(); + stubSetCreateExecTask(); + stubSetAsyncKillTask(); + stubSetDestroyTask(); + stubSetDestroyDataSinker(); + stubSetGetDataLength(); + stubSetEndPut(); + stubSetPutDataBlock(); + stubSetGetDataBlock(); + + srand(time(NULL)); + qwtTestStop = false; + qwtTestQuitThreadNum = 0; + + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + ASSERT_EQ(code, 0); + + qwtTestMaxExecTaskUsec = 0; + qwtTestReqMaxDelayUsec = 1000000; + + tsem_init(&qwtTestQuerySem, 0, 0); + tsem_init(&qwtTestFetchSem, 0, 0); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t t1,t2,t3,t4,t5; + pthread_create(&(t1), &thattr, qwtclientThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, mgmt); + pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); + + while (true) { + if (qwtTestDeadLoop) { + sleep(1); + } else { + sleep(qwtTestMTRunSec); + break; + } + } + + qwtTestStop = true; + + + while (true) { + if (qwtTestQuitThreadNum == 3) { + break; + } + + sleep(1); + + if (qwtTestCaseFinished) { + if (qwtTestQuitThreadNum < 3) { + tsem_post(&qwtTestQuerySem); + tsem_post(&qwtTestFetchSem); + + usleep(10); + } + } + + } + + qwtTestQueryQueueNum = 0; + qwtTestQueryQueueRIdx = 0; + qwtTestQueryQueueWIdx = 0; + qwtTestQueryQueueLock = 0; + qwtTestFetchQueueNum = 0; + qwtTestFetchQueueRIdx = 0; + qwtTestFetchQueueWIdx = 0; + qwtTestFetchQueueLock = 0; + + qWorkerDestroy(&mgmt); +} + + +#if 0 +TEST(rcTest, dropTest) { void *mgmt = NULL; int32_t code = 0; void *mockPointer = (void *)0x1; @@ -759,13 +1330,16 @@ TEST(rcTest, multithread) { code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); + tsem_init(&qwtTestQuerySem, 0, 0); + tsem_init(&qwtTestFetchSem, 0, 0); + pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_t t1,t2,t3,t4,t5; - pthread_create(&(t1), &thattr, controlThread, mgmt); - pthread_create(&(t2), &thattr, queryQueueThread, NULL); - pthread_create(&(t3), &thattr, fetchQueueThread, NULL); + pthread_create(&(t1), &thattr, clientThread, mgmt); + pthread_create(&(t2), &thattr, queryQueueThread, mgmt); + pthread_create(&(t3), &thattr, fetchQueueThread, mgmt); while (true) { if (qwtTestDeadLoop) { @@ -781,12 +1355,13 @@ TEST(rcTest, multithread) { qWorkerDestroy(&mgmt); } - +#endif int main(int argc, char** argv) { + srand(time(NULL)); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 6cc08bc3ad..223fa300df 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -275,10 +275,13 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } -int32_t schRecordTaskSucceedNode(SSchTask *pTask) { - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); - - assert(NULL != addr); +int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { + int32_t idx = atomic_load_8(&pTask->candidateIdx); + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, idx); + if (NULL == addr) { + SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", idx, (int32_t)taosArrayGetSize(pTask->candidateAddrs)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } pTask->succeedAddr = *addr; @@ -578,9 +581,10 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod tsem_post(&pJob->rspSem); } - SCH_ERR_RET(atomic_load_32(&pJob->errCode)); + int32_t code = atomic_load_32(&pJob->errCode); + SCH_ERR_RET(code); - assert(0); + SCH_JOB_ELOG("job errCode is invalid, errCode:%d", code); } @@ -721,7 +725,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); - SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); + SCH_ERR_JRET(schRecordTaskSucceedNode(pJob, pTask)); int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; if (parentNum == 0) { @@ -738,7 +742,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } else if (taskDone > pTask->level->taskNum) { - assert(0); + SCH_TASK_ELOG("taskDone number invalid, done:%d, total:%d", taskDone, pTask->level->taskNum); } if (pTask->level->taskFailed > 0) { @@ -871,18 +875,21 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } atomic_store_ptr(&pJob->res, rsp); - atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows); + atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); } + SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); + SCH_ERR_JRET(schProcessOnDataFetched(pJob)); break; } - case TDMT_VND_DROP_TASK: { + case TDMT_VND_DROP_TASK_RSP: { // SHOULD NEVER REACH HERE - assert(0); + SCH_TASK_ELOG("invalid status to handle drop task rsp, ref:%d", atomic_load_32(&pJob->ref)); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); break; } default: @@ -1030,7 +1037,9 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t qError("QID:%"PRIx64 ",TID:%"PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code); SCH_ERR_JRET(code); } - + + qDebug("QID:%"PRIx64 ",TID:%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); + return TSDB_CODE_SUCCESS; _return: @@ -1288,9 +1297,11 @@ void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->failTasks); } -int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { - if (nodeList && taosArrayGetSize(nodeList) <= 0) { - qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId); +int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { + qDebug("QID:0x%"PRIx64" job started", pDag->queryId); + + if (pNodeList && taosArrayGetSize(pNodeList) <= 0) { + qDebug("QID:0x%"PRIx64" input exec nodeList is empty", pDag->queryId); } int32_t code = 0; @@ -1302,7 +1313,10 @@ int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struc pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; - pJob->nodeList = nodeList; + + if (pNodeList != NULL) { + pJob->nodeList = taosArrayDup(pNodeList); + } SCH_ERR_JRET(schValidateAndBuildJob(pDag, pJob)); @@ -1356,7 +1370,7 @@ _return: *(SSchJob **)job = NULL; - scheduleFreeJob(pJob); + schedulerFreeJob(pJob); SCH_RET(code); } @@ -1401,7 +1415,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1418,12 +1432,12 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru return TSDB_CODE_SUCCESS; } -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, false)); return TSDB_CODE_SUCCESS; } @@ -1498,9 +1512,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { info = NULL; _return: - schedulerFreeTaskList(info); - SCH_RET(code); } @@ -1551,7 +1563,7 @@ _return: } -int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { +int32_t schedulerFetchRows(SSchJob *pJob, void** pData) { if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1616,11 +1628,12 @@ _return: } *pData = rsp; + SCH_JOB_DLOG("empty res and set query complete, code:%x", code); } atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - SCH_JOB_DLOG("fetch done, code:%s", tstrerror(code)); + SCH_JOB_DLOG("fetch done, totalRows:%d, code:%s", pJob->resNumOfRows, tstrerror(code)); atomic_sub_fetch_32(&pJob->ref, 1); SCH_RET(code); @@ -1638,7 +1651,7 @@ int32_t scheduleCancelJob(void *job) { SCH_RET(code); } -void scheduleFreeJob(void *job) { +void schedulerFreeJob(void *job) { if (NULL == job) { return; } @@ -1667,7 +1680,8 @@ void scheduleFreeJob(void *job) { usleep(1); } else { - assert(0); + SCH_JOB_ELOG("invalid job ref number, ref:%d", ref); + break; } } @@ -1706,7 +1720,7 @@ void scheduleFreeJob(void *job) { tfree(pJob); - qDebug("QID:%"PRIx64" job freed", queryId); + qDebug("QID:0x%"PRIx64" job freed", queryId); } void schedulerFreeTaskList(SArray *taskList) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index fb13149713..58714c51cc 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -334,7 +334,7 @@ void schtFreeQueryJob(int32_t freeThread) { SSchJob *job = atomic_load_ptr(&pQueryJob); if (job && atomic_val_compare_exchange_ptr(&pQueryJob, job, NULL)) { - scheduleFreeJob(job); + schedulerFreeJob(job); if (freeThread) { if (++freeNum % schtTestPrintNum == 0) { printf("FreeNum:%d\n", freeNum); @@ -378,7 +378,7 @@ void* schtRunJobThread(void *aa) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &job); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &job); assert(code == 0); execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); @@ -472,7 +472,7 @@ void* schtRunJobThread(void *aa) { atomic_store_32(&schtStartFetch, 1); void *data = NULL; - code = scheduleFetchRows(pQueryJob, &data); + code = schedulerFetchRows(pQueryJob, &data); assert(code == 0 || code); if (0 == code) { @@ -482,7 +482,7 @@ void* schtRunJobThread(void *aa) { } data = NULL; - code = scheduleFetchRows(pQueryJob, &data); + code = schedulerFetchRows(pQueryJob, &data); assert(code == 0 || code); schtFreeQueryJob(0); @@ -539,7 +539,7 @@ TEST(queryTest, normalCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); ASSERT_EQ(code, 0); SSchJob *job = (SSchJob *)pJob; @@ -594,7 +594,7 @@ TEST(queryTest, normalCase) { pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, job); void *data = NULL; - code = scheduleFetchRows(job, &data); + code = schedulerFetchRows(job, &data); ASSERT_EQ(code, 0); SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; @@ -603,11 +603,11 @@ TEST(queryTest, normalCase) { tfree(data); data = NULL; - code = scheduleFetchRows(job, &data); + code = schedulerFetchRows(job, &data); ASSERT_EQ(code, 0); ASSERT_TRUE(data); - scheduleFreeJob(pJob); + schedulerFreeJob(pJob); schtFreeQueryDag(&dag); @@ -649,11 +649,11 @@ TEST(insertTest, normalCase) { pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); SQueryResult res = {0}; - code = scheduleExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); - scheduleFreeJob(pInsertJob); + schedulerFreeJob(pInsertJob); schedulerDestroy(); } diff --git a/source/libs/sync/inc/sync_type.h b/source/libs/sync/inc/sync_type.h index 9c4bc9e63c..c5c4cc3a76 100644 --- a/source/libs/sync/inc/sync_type.h +++ b/source/libs/sync/inc/sync_type.h @@ -43,12 +43,12 @@ typedef struct SSyncRaftLog SSyncRaftLog; typedef struct SSyncRaftEntry SSyncRaftEntry; #if 0 -#ifndef MIN -#define MIN(x, y) (((x) < (y)) ? (x) : (y)) +#ifndef TMIN +#define TMIN(x, y) (((x) < (y)) ? (x) : (y)) #endif -#ifndef MAX -#define MAX(x, y) (((x) > (y)) ? (x) : (y)) +#ifndef TMAX +#define TMAX(x, y) (((x) > (y)) ? (x) : (y)) #endif #endif diff --git a/source/libs/sync/src/sync_raft_progress.c b/source/libs/sync/src/sync_raft_progress.c index a3ab93c0fc..6577972b29 100644 --- a/source/libs/sync/src/sync_raft_progress.c +++ b/source/libs/sync/src/sync_raft_progress.c @@ -59,7 +59,7 @@ bool syncRaftProgressMaybeUpdate(SSyncRaftProgress* progress, SyncIndex lastInde probeAcked(progress); } - progress->nextIndex = MAX(progress->nextIndex, lastIndex + 1); + progress->nextIndex = TMAX(progress->nextIndex, lastIndex + 1); return updated; } @@ -100,7 +100,7 @@ bool syncRaftProgressMaybeDecrTo(SSyncRaftProgress* progress, return false; } - progress->nextIndex = MAX(MIN(rejected, matchHint + 1), 1); + progress->nextIndex = TMAX(TMIN(rejected, matchHint + 1), 1); progress->probeSent = false; return true; @@ -166,7 +166,7 @@ void syncRaftProgressBecomeProbe(SSyncRaftProgress* progress) { if (progress->state == PROGRESS_STATE_SNAPSHOT) { SyncIndex pendingSnapshotIndex = progress->pendingSnapshotIndex; resetProgressState(progress, PROGRESS_STATE_PROBE); - progress->nextIndex = MAX(progress->matchIndex + 1, pendingSnapshotIndex + 1); + progress->nextIndex = TMAX(progress->matchIndex + 1, pendingSnapshotIndex + 1); } else { resetProgressState(progress, PROGRESS_STATE_PROBE); progress->nextIndex = progress->matchIndex + 1; diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index eb63f2b144..3cb5a65572 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ -set(TDB_SUBDIRS "btree" "db" "hash" "mpool" "dmgr") +set(TDB_SUBDIRS "db") foreach(TDB_SUBDIR ${TDB_SUBDIRS}) aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC) endforeach() diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index eee8f8ed33..faf9208f8f 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,30 +22,41 @@ extern "C" { #endif -#define TDB_EXTERN -#define TDB_PUBLIC -#define TDB_STATIC static +// #define TDB_EXTERN +// #define TDB_PUBLIC +// #define TDB_STATIC static -typedef enum { - TDB_BTREE_T = 0, - TDB_HASH_T, - TDB_HEAP_T, -} tdb_db_t; +// typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; -// Forward declaration -typedef struct TDB TDB; -typedef struct TDB_CURSOR TDB_CURSOR; +// // Forward declarations +// typedef struct TDB TDB; +// // typedef struct TDB_MPOOL TDB_MPOOL; +// // typedef struct TDB_MPFILE TDB_MPFILE; +// // typedef struct TDB_CURSOR TDB_CURSOR; -// SKey -typedef struct { - void* bdata; - uint32_t size; -} TDB_KEY, TDB_VALUE; +// typedef struct { +// void* bdata; +// uint32_t size; +// } TDB_KEY, TDB_VALUE; -// TDB Operations -TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type); -TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); -TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags); +// // TDB Operations +// int tdbCreateDB(TDB** dbpp, tdb_db_t type); +// int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); +// int tdbCloseDB(TDB* dbp, uint32_t flags); +// int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); +// int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); + +// // TDB_MPOOL +// int tdbOpenMPool(TDB_MPOOL** mp); +// int tdbCloseMPool(TDB_MPOOL* mp); + +// // TDB_MPFILE +// int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp); +// int tdbCloseMPFile(TDB_MPFILE** mpf); + +// // TDB_CURSOR +// int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp); +// int tdbCloseCurosr(TDB_CURSOR* tdbcp); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/db/tdbDB.c b/source/libs/tdb/src/db/tdbDB.c deleted file mode 100644 index eaf85ea4a1..0000000000 --- a/source/libs/tdb/src/db/tdbDB.c +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbDB.h" -#include "tdb.h" - -TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type) { - TDB* dbp; - int ret; - - dbp = calloc(1, sizeof(*dbp)); - if (dbp == NULL) { - return -1; - } - - dbp->pageSize = TDB_DEFAULT_PGSIZE; - dbp->type = type; - - switch (type) { - case TDB_BTREE_T: - // ret = tdbInitBtreeDB(dbp); - // if (ret < 0) goto _err; - break; - case TDB_HASH_T: - // ret = tdbInitHashDB(dbp); - // if (ret < 0) goto _err; - break; - case TDB_HEAP_T: - // ret = tdbInitHeapDB(dbp); - // if (ret < 0) goto _err; - break; - default: - break; - } - - *dbpp = dbp; - return 0; - -_err: - if (dbp) { - free(dbp); - } - *dbpp = NULL; - return 0; -} - -TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags) { - int ret = 0; - - if ((dbp->fname = strdup(fname)) == NULL) { - ret = -1; - return ret; - } - - // Create the backup file if the file not exists - - // Open the file as a sub-db or a master-db - if (dbname) { - if ((dbp->dbname = strdup(dbname)) == NULL) { - ret = -1; - return ret; - } - // TODO: Open the DB as a SUB-DB in this file - } else { - // TODO: Open the DB as a MASTER-DB in this file - } - - return ret; -} - -TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) { - // TODO - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbHeap.h b/source/libs/tdb/src/db/tdb_db.c similarity index 73% rename from source/libs/tdb/src/inc/tdbHeap.h rename to source/libs/tdb/src/db/tdb_db.c index 25a812fa5f..3675844535 100644 --- a/source/libs/tdb/src/inc/tdbHeap.h +++ b/source/libs/tdb/src/db/tdb_db.c @@ -13,23 +13,14 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_HEAP_H_ -#define _TD_TDB_HEAP_H_ +#include "tdb_db.h" -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { +int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags) { // TODO -} TDB_HEAP; - -TDB_PUBLIC int tdbInitHeapDB(TDB *dbp); - -#ifdef __cplusplus + return 0; } -#endif -#endif /*_TD_TDB_HEAP_H_*/ \ No newline at end of file +int tdbClose(TDB *dbp, uint32_t flags) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c new file mode 100644 index 0000000000..9b67f405a9 --- /dev/null +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdb_mpool.h" + +static int tdbGnrtFileID(const char *fname, uint8_t *fileid); +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp); + +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { + TDB_MPOOL *mp = NULL; + size_t tsize; + pg_t * pagep; + + // check parameters + if (!TDB_IS_PGSIZE_VLD(pgsize)) { + tdbError("invalid page size"); + return -1; + } + + // allocate handle + mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); + if (mp == NULL) { + tdbError("failed to malloc memory pool handle"); + goto _err; + } + + // initialize the handle + mp->cachesize = cachesize; + mp->pgsize = pgsize; + mp->npages = cachesize / pgsize; + mp->clockHand = 0; + + TD_DLIST_INIT(&mp->freeList); + + mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t)); + if (mp->pages == NULL) { + tdbError("failed to malloc memory pool pages"); + goto _err; + } + + for (frame_id_t i = 0; i < mp->npages; i++) { + mp->pages[i].p = malloc(pgsize); + if (mp->pages[i].p == NULL) { + goto _err; + } + + taosInitRWLatch(&mp->pages[i].rwLatch); + mp->pages[i].frameid = i; + mp->pages[i].pgid = TDB_IVLD_PGID; + + // add new page to the free list + TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free); + } + +#define PGTAB_FACTOR 1.0 + mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR; + mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t)); + if (mp->pgtab.hashtab == NULL) { + tdbError("failed to malloc memory pool hash table"); + goto _err; + } + + // return + *mpp = mp; + return 0; + +_err: + tdbMPoolClose(mp); + *mpp = NULL; + return -1; +} + +int tdbMPoolClose(TDB_MPOOL *mp) { + if (mp) { + tfree(mp->pgtab.hashtab); + if (mp->pages) { + for (int i = 0; i < mp->npages; i++) { + tfree(mp->pages[i].p); + } + + free(mp->pages); + } + + free(mp); + } + return 0; +} + +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { + TDB_MPFILE *mpf; + + if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) { + return -1; + } + + mpf->fd = -1; + + if ((mpf->fname = strdup(fname)) == NULL) { + goto _err; + } + + if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) { + goto _err; + } + + if (tdbGnrtFileID(fname, mpf->fileid) < 0) { + goto _err; + } + + // Register current MPF to MP + tdbMPoolRegFile(mp, mpf); + + *mpfp = mpf; + return 0; + +_err: + tdbMPoolFileClose(mpf); + *mpfp = NULL; + return -1; +} + +int tdbMPoolFileClose(TDB_MPFILE *mpf) { + if (mpf) { + if (mpf->fd > 0) { + close(mpf->fd); + } + tfree(mpf->fname); + free(mpf); + } + return 0; +} + +#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \ + ({ \ + uint64_t *tmp = (uint64_t *)fileid; \ + (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ + }) + +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { + pg_t * pagep; + TDB_MPOOL *mp; + pg_list_t *pglist; + + mp = mpf->mp; + + // check if the page already in pool + pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); + pagep = TD_DLIST_HEAD(pglist); + while (pagep) { + if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) { + break; + } + + pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash); + } + + if (pagep) { + // page is found + // todo: pin the page and return + *(void **)addr = pagep->p; + return 0; + } + + // page not found + pagep = TD_DLIST_HEAD(&mp->freeList); + if (pagep) { + // has free page + TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); + } else { + // no free page available + tdbMPoolClockEvictPage(mp, &pagep); + if (pagep) { + if (pagep->dirty) { + // TODO: Handle dirty page eviction + } + } + } + + if (pagep == NULL) { + // no available container page + return -1; + } + + // load page from the disk if a container page is available + // TODO: load the page from the disk + if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) { + return -1; + } + + memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN); + pagep->pgid.pgno = pgno; + pagep->dirty = 0; + pagep->pinRef = 1; + + // add current page to page table + TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); + + return 0; +} + +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { + // TODO + return 0; +} + +static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { + struct stat statbuf; + + if (stat(fname, &statbuf) < 0) { + return -1; + } + + memset(fileid, 0, TDB_FILE_ID_LEN); + + ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino; + ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev; + ((uint64_t *)fileid)[2] = rand(); + + return 0; +} + +#define MPF_GET_BUCKETID(fileid) \ + ({ \ + uint64_t *tmp = (uint64_t *)fileid; \ + (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ + }) + +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + mpf_bucket_t *bktp; + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); + + taosWLockLatch(&(bktp->latch)); + + TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node); + + taosWUnLockLatch(&(bktp->latch)); + + mpf->mp = mp; +} + +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { + TDB_MPFILE * mpf = NULL; + mpf_bucket_t *bktp; + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid); + + taosRLockLatch(&(bktp->latch)); + + mpf = TD_DLIST_HEAD(bktp); + while (mpf) { + if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) { + break; + } + + mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node); + } + + taosRUnLockLatch(&(bktp->latch)); + + return mpf; +} + +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + mpf_bucket_t *bktp; + TDB_MPFILE * tmpf; + + if (mpf->mp == NULL) return; + + ASSERT(mpf->mp == mp); + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); + + taosWLockLatch(&(bktp->latch)); + + tmpf = TD_DLIST_HEAD(bktp); + + while (tmpf) { + if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) { + TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node); + break; + } + + tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node); + } + + taosWUnLockLatch(&(bktp->latch)); + + ASSERT(tmpf == mpf); +} + +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { + pgsize_t pgsize; + TDB_MPOOL *mp; + off_t offset; + size_t rsize; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + // TODO: use loop to read all data + rsize = pread(mpf->fd, p, pgsize, offset); + // TODO: error handle + + return 0; +} + +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { + pgsize_t pgsize; + TDB_MPOOL *mp; + off_t offset; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + lseek(mpf->fd, offset, SEEK_SET); + // TODO: handle error + + write(mpf->fd, p, pgsize); + // TODO: handle error + + return 0; +} + +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) { + pg_t * pagep; + frame_id_t och; + + *pagepp = NULL; + och = mp->clockHand; + + do { + pagep = mp->pages + mp->clockHand; + mp->clockHand = (mp->clockHand + 1) % mp->npages; + + if (pagep->pinRef == 0) { + if (pagep->rbit == 1) { + pagep->rbit = 0; + } else { + break; + } + } + + if (mp->clockHand == och) { + return; + } + } while (1); + + *pagepp = pagep; +} \ No newline at end of file diff --git a/source/libs/tdb/src/dmgr/tdbDiskMgr.c b/source/libs/tdb/src/dmgr/tdbDiskMgr.c deleted file mode 100644 index 71ab5f2589..0000000000 --- a/source/libs/tdb/src/dmgr/tdbDiskMgr.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbDiskMgr.h" - -struct STkvDiskMgr { - char * fname; - uint16_t pgsize; - FileFd fd; - pgid_t npgid; -}; - -#define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) - -int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - STkvDiskMgr *pDiskMgr; - - pDiskMgr = malloc(sizeof(*pDiskMgr)); - if (pDiskMgr == NULL) { - return -1; - } - - pDiskMgr->fname = strdup(fname); - if (pDiskMgr->fname == NULL) { - free(pDiskMgr); - return -1; - } - pDiskMgr->pgsize = pgsize; - pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755); - if (pDiskMgr->fd < 0) { - free(pDiskMgr->fname); - free(pDiskMgr); - return -1; - } - - *ppDiskMgr = pDiskMgr; - - return 0; -} - -int tdmClose(STkvDiskMgr *pDiskMgr) { - close(pDiskMgr->fd); - free(pDiskMgr->fname); - free(pDiskMgr); - return 0; -} - -int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { - taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); - taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); - return 0; -} - -int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { - taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); - taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); - return 0; -} - -int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } - -int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h deleted file mode 100644 index 28258b8e60..0000000000 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_BTREE_H_ -#define _TD_TDB_BTREE_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - pgid_t root; // root page number -} TDB_BTREE; - -TDB_PUBLIC int tdbInitBtreeDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBufPool.h b/source/libs/tdb/src/inc/tdbBufPool.h deleted file mode 100644 index 5200d22faa..0000000000 --- a/source/libs/tdb/src/inc/tdbBufPool.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_BUF_POOL_H_ -#define _TD_TDB_BUF_POOL_H_ - -#include "tdbPage.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct STdbBufPool STdbBufPool; - -int tbpOpen(STdbBufPool **ppTkvBufPool); -int tbpClose(STdbBufPool *pTkvBufPool); -STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool); -int tbpDelPage(STdbBufPool *pTkvBufPool); -STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid); -int tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid); -void tbpFlushPages(STdbBufPool *pTkvBufPool); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_BUF_POOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDiskMgr.h b/source/libs/tdb/src/inc/tdbDiskMgr.h deleted file mode 100644 index b83a147437..0000000000 --- a/source/libs/tdb/src/inc/tdbDiskMgr.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDISK_MGR_H_ -#define _TD_TDISK_MGR_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" - -#include "tdbDef.h" - -typedef struct STkvDiskMgr STkvDiskMgr; - -int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(STkvDiskMgr *pDiskMgr); -int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int tdmFlush(STkvDiskMgr *pDiskMgr); -pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDISK_MGR_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbHash.h b/source/libs/tdb/src/inc/tdbHash.h deleted file mode 100644 index 8219bda2f8..0000000000 --- a/source/libs/tdb/src/inc/tdbHash.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_HASH_H_ -#define _TD_TDB_HASH_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - // TODO -} TDB_HASH; - -TDB_PUBLIC int tdbInitHashDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_HASH_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h deleted file mode 100644 index e7245b6c39..0000000000 --- a/source/libs/tdb/src/inc/tdbPage.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TKV_PAGE_H_ -#define _TD_TKV_PAGE_H_ - -#include "os.h" -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - pgid_t pgid; - int32_t pinCount; - bool idDirty; - char* pData; -} STdbPage; - -typedef struct { - uint16_t dbver; - uint16_t pgsize; - uint32_t cksm; -} STdbPgHdr; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TKV_PAGE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdb_db.h similarity index 64% rename from source/libs/tdb/src/inc/tdbDB.h rename to source/libs/tdb/src/inc/tdb_db.h index d0ef9e22d0..5f0529462b 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdb_db.h @@ -16,38 +16,28 @@ #ifndef _TD_TDB_DB_H_ #define _TD_TDB_DB_H_ -#include "tdb.h" -#include "tdbBtree.h" -#include "tdbHash.h" -#include "tdbHeap.h" +#include "tdb_mpool.h" #ifdef __cplusplus extern "C" { #endif -typedef struct { - // TODO -} TDB_MPOOL; - -typedef struct { - int fd; -} TDB_FH; +typedef struct TDB TDB; struct TDB { - pgsize_t pageSize; - tdb_db_t type; - char * fname; - char * dbname; - union { - TDB_BTREE *btree; - TDB_HASH * hash; - TDB_HEAP * heap; - } dbam; // db access method - - TDB_FH * fhp; // The backup file handle - TDB_MPOOL *mph; // The memory pool handle + char * fname; + char * dbname; + TDB_MPFILE *mpf; + // union { + // TDB_BTREE *btree; + // TDB_HASH * hash; + // TDB_HEAP * heap; + // } dbam; // db access method }; +int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags); +int tdbClose(TDB *dbp, uint32_t flags); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDef.h b/source/libs/tdb/src/inc/tdb_inc.h similarity index 65% rename from source/libs/tdb/src/inc/tdbDef.h rename to source/libs/tdb/src/inc/tdb_inc.h index 4b5e54368b..885191477c 100644 --- a/source/libs/tdb/src/inc/tdbDef.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -13,18 +13,30 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_DEF_H_ -#define _TD_TDB_DEF_H_ +#ifndef _TD_TDB_INC_H_ +#define _TD_TDB_INC_H_ #include "os.h" +#include "tlist.h" +#include "tlockfree.h" #ifdef __cplusplus extern "C" { #endif +// pgno_t +typedef int32_t pgno_t; +#define TDB_IVLD_PGNO ((pgno_t)-1) + +// fileid +#define TDB_FILE_ID_LEN 24 + // pgid_t -typedef int32_t pgid_t; -#define TDB_IVLD_PGID ((pgid_t)-1) +typedef struct { + uint8_t fileid[TDB_FILE_ID_LEN]; + pgno_t pgno; +} pgid_t; +#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; // framd_id_t typedef int32_t frame_id_t; @@ -34,10 +46,13 @@ typedef int32_t pgsize_t; #define TDB_MIN_PGSIZE 512 #define TDB_MAX_PGSIZE 16384 #define TDB_DEFAULT_PGSIZE 4096 -#define TDB_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE)) +#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) + +// tdb_log +#define tdbError(var) #ifdef __cplusplus } #endif -#endif /*_TD_TDB_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TDB_INC_H_*/ diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h new file mode 100644 index 0000000000..37c82f3833 --- /dev/null +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDB_MPOOL_H_ +#define _TD_TDB_MPOOL_H_ + +#include "tdb_inc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Exposed handle +typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPFILE TDB_MPFILE; + +typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; +typedef struct pg_t { + SRWLatch rwLatch; + frame_id_t frameid; + pgid_t pgid; + uint8_t dirty; + uint8_t rbit; + int32_t pinRef; + pg_free_dlist_node_t free; + pg_hash_dlist_node_t hash; + void * p; +} pg_t; + +typedef TD_DLIST(pg_t) pg_list_t; +typedef struct { + SRWLatch latch; + TD_DLIST(TDB_MPFILE); +} mpf_bucket_t; +struct TDB_MPOOL { + int64_t cachesize; + pgsize_t pgsize; + int32_t npages; + pg_t * pages; + pg_list_t freeList; + frame_id_t clockHand; + struct { + int32_t nbucket; + pg_list_t *hashtab; + } pgtab; // page table, hash + struct { +#define MPF_HASH_BUCKETS 16 + mpf_bucket_t buckets[MPF_HASH_BUCKETS]; + } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table +}; + +#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] + +typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; +struct TDB_MPFILE { + char * fname; // file name + int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL * mp; // underlying memory pool + td_mpf_dlist_node_t node; +}; + +/*=================================================== Exposed apis ==================================================*/ +// TDB_MPOOL +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); +int tdbMPoolClose(TDB_MPOOL *mp); +int tdbMPoolSync(TDB_MPOOL *mp); + +// TDB_MPFILE +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPoolFileClose(TDB_MPFILE *mpf); +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFileSync(TDB_MPFILE *mpf); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/mpool/tdbBufPool.c b/source/libs/tdb/src/mpool/tdbBufPool.c deleted file mode 100644 index bc3c386b0f..0000000000 --- a/source/libs/tdb/src/mpool/tdbBufPool.c +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "thash.h" -#include "tlist.h" - -#include "tdbBufPool.h" -#include "tdbDiskMgr.h" -#include "tdbPage.h" - -struct SFrameIdWrapper { - TD_SLIST_NODE(SFrameIdWrapper); - frame_id_t id; -}; - -struct STdbBufPool { - STdbPage* pages; - STkvDiskMgr* pDiskMgr; - SHashObj* pgTb; // page_id_t --> frame_id_t - TD_SLIST(SFrameIdWrapper) freeList; - pthread_mutex_t mutex; -}; - -typedef struct STkvLRUReplacer { -} STkvLRUReplacer; - -typedef struct STkvLFUReplacer { -} STkvLFUReplacer; - -typedef struct STkvCLKReplacer { -} STkvCLKReplacer; - -typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t; - -typedef struct STkvReplacer { - tkv_replacer_t type; - union { - STkvLRUReplacer lruRep; - STkvLFUReplacer lfuRep; - STkvCLKReplacer clkRep; - }; -} STkvReplacer; \ No newline at end of file diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index 2d77c1f4e9..7fbfaf5506 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -1,3 +1,7 @@ +# tdbMPoolTest +add_executable(tdbMPoolTest "tdbMPoolTest.cpp") +target_link_libraries(tdbMPoolTest tdb gtest gtest_main) + # tdbTest add_executable(tdbTest "tdbTest.cpp") target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file diff --git a/source/libs/tdb/test/tdbMPoolTest.cpp b/source/libs/tdb/test/tdbMPoolTest.cpp new file mode 100644 index 0000000000..17381759fb --- /dev/null +++ b/source/libs/tdb/test/tdbMPoolTest.cpp @@ -0,0 +1,31 @@ +#include "gtest/gtest.h" + +#include + +#include "tdb_mpool.h" + +TEST(tdb_mpool_test, test1) { + TDB_MPOOL * mp; + TDB_MPFILE *mpf; + pgno_t pgno; + void * pgdata; + + // open mp + tdbMPoolOpen(&mp, 16384, 4096); + + // open mpf + tdbMPoolFileOpen(&mpf, "test.db", mp); + +#define TEST1_TOTAL_PAGES 100 + for (int i = 0; i < TEST1_TOTAL_PAGES; i++) { + tdbMPoolFileNewPage(mpf, &pgno, pgdata); + + *(pgno_t *)pgdata = i; + } + + // close mpf + tdbMPoolFileClose(mpf); + + // close mp + tdbMPoolClose(mp); +} diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index f27e17f1ca..113bb2560f 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -3,8 +3,8 @@ #include "tdb.h" TEST(tdb_api_test, tdb_create_open_close_db_test) { - int ret; - TDB *dbp; + // int ret; + // TDB *dbp; // tdbCreateDB(&dbp, TDB_BTREE_T); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index c760acd52e..f0b54a82e7 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -123,9 +123,9 @@ typedef struct { } SRpcReqContext; typedef struct { - SRpcInfo* pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app + SRpcInfo* pTransInst; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app // struct SRpcConn* pConn; // pConn allocated tmsg_t msgType; // message type uint8_t* pCont; // content provided by app diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index d870ae98ab..3c58a76a44 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -42,6 +42,8 @@ int tsRpcMaxRetry; int tsRpcHeadSize; int tsRpcOverhead; +SHashObj *tsFqdnHash; + #ifndef USE_UV typedef struct { @@ -215,6 +217,8 @@ static void rpcInitImp(void) { tsRpcOverhead = sizeof(SRpcReqContext); tsRpcRefId = taosOpenRef(200, rpcFree); + + tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); } int32_t rpcInit(void) { @@ -224,6 +228,9 @@ int32_t rpcInit(void) { void rpcCleanup(void) { taosCloseRef(tsRpcRefId); + taosHashClear(tsFqdnHash); + taosHashCleanup(tsFqdnHash); + tsFqdnHash = NULL; tsRpcRefId = -1; } @@ -571,7 +578,17 @@ static void rpcFreeMsg(void *msg) { static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) { SRpcConn *pConn; - uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn); + uint32_t peerIp = 0; + uint32_t *pPeerIp = taosHashGet(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1); + if (pPeerIp != NULL) { + peerIp = *pPeerIp; + } else { + peerIp = taosGetIpv4FromFqdn(peerFqdn); + if (peerIp != 0xFFFFFFFF) { + taosHashPut(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1, &peerIp, sizeof(peerIp)); + } + } + if (peerIp == 0xFFFFFFFF) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); terrno = TSDB_CODE_RPC_FQDN_ERROR; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index bfadfe56ef..67afd46b1c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -30,6 +30,7 @@ typedef struct SCliConn { char spi; char secured; uint64_t expireTime; + int8_t notifyCount; // timers already notify to client } SCliConn; typedef struct SCliMsg { @@ -72,8 +73,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); -// process data read from server, auth/decompress etc later -static void clientHandleResp(SCliConn* conn); // check whether already read complete packet from server static bool clientReadComplete(SConnBuffer* pBuf); // alloc buf for read @@ -88,10 +87,15 @@ static void clientAsyncCb(uv_async_t* handle); static void clientDestroy(uv_handle_t* handle); static void clientConnDestroy(SCliConn* pConn, bool clear /*clear tcp handle or not*/); -static void clientMsgDestroy(SCliMsg* pMsg); +// process data read from server, auth/decompress etc later +static void clientHandleResp(SCliConn* conn); +// handle except about conn +static void clientHandleExcept(SCliConn* conn); // handle req from app static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void clientMsgDestroy(SCliMsg* pMsg); +static void destroyTransConnCtx(STransConnCtx* ctx); // thread obj static SCliThrdObj* createThrdObj(); static void destroyThrdObj(SCliThrdObj* pThrd); @@ -100,22 +104,38 @@ static void* clientThread(void* arg); static void clientHandleResp(SCliConn* conn) { STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; - SRpcInfo* pRpc = pCtx->pRpc; + SRpcInfo* pRpc = pCtx->pTransInst; SRpcMsg rpcMsg; rpcMsg.pCont = conn->readBuf.buf; rpcMsg.contLen = conn->readBuf.len; rpcMsg.ahandle = pCtx->ahandle; (pRpc->cfp)(NULL, &rpcMsg, NULL); + conn->notifyCount += 1; SCliThrdObj* pThrd = conn->hostThrd; addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + + // start thread's timer of conn pool if not active if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) { uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } - free(pCtx->ip); - free(pCtx); - // impl + destroyTransConnCtx(pCtx); +} +static void clientHandleExcept(SCliConn* pConn) { + SCliMsg* pMsg = pConn->data; + + STransConnCtx* pCtx = pMsg->ctx; + SRpcInfo* pRpc = pCtx->pTransInst; + + SRpcMsg rpcMsg; + rpcMsg.ahandle = pCtx->ahandle; + // SRpcInfo* pRpc = pMsg->ctx->pRpc; + (pRpc->cfp)(NULL, &rpcMsg, NULL); + + pConn->notifyCount += 1; + destroyTransConnCtx(pCtx); + clientConnDestroy(pConn, true); } static void clientTimeoutCb(uv_timer_t* handle) { @@ -191,6 +211,7 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); + conn->notifyCount = 0; // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); @@ -246,19 +267,21 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf if (nread > 0) { pBuf->len += nread; if (clientReadComplete(pBuf)) { - tDebug("alread read complete"); + tDebug("conn %p read complete", conn); clientHandleResp(conn); } else { - tDebug("read half packet, continue to read"); + tDebug("conn %p read partial packet, continue to read", conn); } return; } assert(nread <= 0); if (nread == 0) { + tError("conn %p closed", conn); return; } - if (nread != UV_EOF) { - tDebug("read error %s", uv_err_name(nread)); + if (nread < 0) { + tError("conn %p read error: %s", conn, uv_err_name(nread)); + clientHandleExcept(conn); } // tDebug("Read error %s\n", uv_err_name(nread)); // uv_close((uv_handle_t*)handle, clientDestroy); @@ -282,19 +305,20 @@ static void clientDestroy(uv_handle_t* handle) { static void clientWriteCb(uv_write_t* req, int status) { SCliConn* pConn = req->data; + if (status == 0) { - tDebug("data already was written on stream"); + tDebug("conn %p data already was written out", pConn); } else { - tError("failed to write: %s", uv_err_name(status)); - clientConnDestroy(pConn, true); + tError("conn %p failed to write: %s", pConn, uv_err_name(status)); + clientHandleExcept(pConn); return; } SCliThrdObj* pThrd = pConn->hostThrd; - if (pConn->stream == NULL) { - pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); - uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); - pConn->stream->data = pConn; - } + // if (pConn->stream == NULL) { + // pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); + // uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); + // pConn->stream->data = pConn; + //} uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb); // impl later } @@ -310,30 +334,19 @@ static void clientWrite(SCliConn* pConn) { pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("data write out, msgType : %d, len: %d", pHead->msgType, msgLen); + tDebug("conn %p data write out, msgType : %d, len: %d", pConn, pHead->msgType, msgLen); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static void clientConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; - SCliMsg* pMsg = pConn->data; - - STransConnCtx* pCtx = pMsg->ctx; - SRpcInfo* pRpc = pCtx->pRpc; - if (status != 0) { // tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); - tError("failed to connect server, errmsg: %s", uv_strerror(status)); - // call user fp later - SRpcMsg rpcMsg; - rpcMsg.ahandle = pCtx->ahandle; - // SRpcInfo* pRpc = pMsg->ctx->pRpc; - (pRpc->cfp)(NULL, &rpcMsg, NULL); - - clientConnDestroy(pConn, true); - // uv_close((uv_handle_t*)req->handle, clientDestroy); + tError("conn %p failed to connect server: %s", pConn, uv_strerror(status)); + clientHandleExcept(pConn); return; } + tDebug("conn %p create", pConn); assert(pConn->stream == req->handle); clientWrite(pConn); @@ -349,6 +362,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { // impl later + tDebug("conn %p get from conn pool", conn); conn->data = pMsg; conn->writeReq->data = conn; @@ -462,6 +476,13 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { free(pThrd->loop); free(pThrd); } + +static void destroyTransConnCtx(STransConnCtx* ctx) { + if (ctx != NULL) { + free(ctx->ip); + } + free(ctx); +} // void taosCloseClient(void* arg) { // impl later @@ -472,7 +493,6 @@ void taosCloseClient(void* arg) { free(cli->pThreadObj); free(cli); } - void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { // impl later char* ip = (char*)(pEpSet->fqdn[pEpSet->inUse]); @@ -487,7 +507,7 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); - pCtx->pRpc = (SRpcInfo*)shandle; + pCtx->pTransInst = (SRpcInfo*)shandle; pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; pCtx->ip = strdup(ip); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index b519a35f24..34e621f0f3 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -16,6 +16,7 @@ #ifdef USE_UV #include "transComm.h" + typedef struct SConn { uv_tcp_t* pTcp; uv_write_t* pWriter; @@ -26,7 +27,6 @@ typedef struct SConn { int ref; int persist; // persist connection or not SConnBuffer connBuf; // read buf, - int count; int inType; void* pTransInst; // rpc init void* ahandle; // @@ -226,7 +226,7 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { tDebug("%p timeout since no activity", conn); } -static void uvProcessData(SConn* pConn) { +static void uvHandleReq(SConn* pConn) { SRecvInfo info; SRecvInfo* p = &info; SConnBuffer* pBuf = &pConn->connBuf; @@ -271,6 +271,7 @@ static void uvProcessData(SConn* pConn) { rpcMsg.ahandle = NULL; rpcMsg.handle = pConn; + pConn->ref++; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth @@ -283,20 +284,23 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { SConnBuffer* pBuf = &conn->connBuf; if (nread > 0) { pBuf->len += nread; - tDebug("on read %p, total read: %d, current read: %d", cli, pBuf->len, (int)nread); + tDebug("conn %p read summroy, total read: %d, current read: %d", conn, pBuf->len, (int)nread); if (readComplete(pBuf)) { - tDebug("alread read complete packet"); - uvProcessData(conn); + tDebug("conn %p alread read complete packet", conn); + uvHandleReq(conn); } else { - tDebug("read half packet, continue to read"); + tDebug("conn %p read partial packet, continue to read", conn); } return; } if (nread == 0) { + tDebug("conn %p except read", conn); + // destroyConn(conn, true); return; } if (nread != UV_EOF) { - tDebug("read error %s", uv_err_name(nread)); + tDebug("conn %p read error: %s", conn, uv_err_name(nread)); + destroyConn(conn, true); } } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { @@ -306,7 +310,8 @@ void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b void uvOnTimeoutCb(uv_timer_t* handle) { // opt - tDebug("time out"); + SConn* pConn = handle->data; + tDebug("conn %p time out", pConn); } void uvOnWriteCb(uv_write_t* req, int status) { @@ -317,9 +322,9 @@ void uvOnWriteCb(uv_write_t* req, int status) { memset(buf->buf, 0, buf->cap); buf->left = -1; if (status == 0) { - tDebug("data already was written on stream"); + tDebug("conn %p data already was written on stream", conn); } else { - tDebug("failed to write data, %s", uv_err_name(status)); + tDebug("conn %p failed to write data, %s", conn, uv_err_name(status)); destroyConn(conn, true); } // opt @@ -334,7 +339,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { // impl later; - tDebug("prepare to send back"); + tDebug("conn %p prepare to send resp", conn); SRpcMsg* pMsg = &conn->sendMsg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); @@ -427,6 +432,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { assert(pending == UV_TCP); SConn* pConn = createConn(); + pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ pConn->pTimer = malloc(sizeof(uv_timer_t)); @@ -448,7 +454,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); - tDebug("new connection created: %d", fd); + tDebug("conn %p created, fd: %d", pConn, fd); uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); } else { tDebug("failed to create new connection"); @@ -515,19 +521,19 @@ void* workerThread(void* arg) { static SConn* createConn() { SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); + ++pConn->ref; return pConn; } -static void connCloseCb(uv_handle_t* handle) { - // impl later - // -} + static void destroyConn(SConn* conn, bool clear) { if (conn == NULL) { return; } + if (--conn->ref == 0) { + return; + } if (clear) { - uv_handle_t handle = *((uv_handle_t*)conn->pTcp); - uv_close(&handle, NULL); + uv_close((uv_handle_t*)conn->pTcp, NULL); } uv_timer_stop(conn->pTimer); free(conn->pTimer); @@ -646,6 +652,7 @@ void rpcSendResponse(const SRpcMsg* pMsg) { pthread_mutex_lock(&pThrd->connMtx); QUEUE_PUSH(&pThrd->conn, &pConn->queue); pthread_mutex_unlock(&pThrd->connMtx); + tDebug("conn %p start to send resp", pConn); uv_async_send(pConn->pWorkerAsync); } diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index fd3496cc17..84814f39fc 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -63,7 +63,7 @@ static void *sendRequest(void *param) { if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); - tDebug("recv response"); + tDebug("recv response succefully"); // usleep(100000000); } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index a3894ceedd..2d6fb8fc76 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -66,7 +66,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { struct stat statbuf; stat(fnameStr, &statbuf); - int readSize = MIN(WAL_MAX_SIZE + 2, statbuf.st_size); + int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); pLastFileInfo->fileSize = statbuf.st_size; FileFd fd = taosOpenFileRead(fnameStr); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 070fe4a6c8..04efbef5b3 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -98,7 +98,7 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) { } if (fileSec <= 100) continue; - int32_t days = (int32_t)(ABS(sec - fileSec) / 86400 + 1); + int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { (void)remove(filename); //printf("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a67d66efb0..56919ff99e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -248,6 +248,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") // mnode-trans TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") @@ -353,7 +354,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_INPUT, "invalid input") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SCH_NOT_EXIST, "Scheduler not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_NOT_EXIST, "Task not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_ALREADY_EXIST, "Task already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST, "Task result cache not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST, "Task context not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLED, "Task cancelled") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_DROPPED, "Task dropped") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLING, "Task cancelling") diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 2b013bfdd0..0b0d8c4c58 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -67,7 +67,7 @@ static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) { } static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = MIN(length, HASH_MAX_CAPACITY); + int32_t len = TMIN(length, HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c284efbcd4..c7b1350591 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -185,7 +185,7 @@ static void taosKeepOldLog(char *oldName) { } } - taosRemoveOldFiles(tsLogDir, ABS(tsLogKeepDays)); + taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays)); } static void *taosThreadToOpenNewFile(void *param) { diff --git a/source/util/src/tstrbuild.c b/source/util/src/tstrbuild.c index eec21d1835..230bff42f5 100644 --- a/source/util/src/tstrbuild.c +++ b/source/util/src/tstrbuild.c @@ -70,11 +70,11 @@ void taosStringBuilderAppendNull(SStringBuilder* sb) { taosStringBuilderAppendSt void taosStringBuilderAppendInteger(SStringBuilder* sb, int64_t v) { char buf[64]; size_t len = snprintf(buf, sizeof(buf), "%" PRId64, v); - taosStringBuilderAppendStringLen(sb, buf, MIN(len, sizeof(buf))); + taosStringBuilderAppendStringLen(sb, buf, TMIN(len, sizeof(buf))); } void taosStringBuilderAppendDouble(SStringBuilder* sb, double v) { char buf[512]; size_t len = snprintf(buf, sizeof(buf), "%.9lf", v); - taosStringBuilderAppendStringLen(sb, buf, MIN(len, sizeof(buf))); + taosStringBuilderAppendStringLen(sb, buf, TMIN(len, sizeof(buf))); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bb6569deb0..a27f5326c9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -2,17 +2,17 @@ #======================b1-start=============== # ---- user -./test.sh -f sim/user/basic1.sim +./test.sh -f tsim/user/basic1.sim # ---- db -./test.sh -f sim/db/basic1.sim -./test.sh -f sim/db/basic6.sim -./test.sh -f sim/db/error1.sim +./test.sh -f tsim/db/basic1.sim +./test.sh -f tsim/db/basic6.sim +./test.sh -f tsim/db/error1.sim # ---- table -./test.sh -f sim/table/basic1.sim +./test.sh -f tsim/table/basic1.sim # ---- dnode -./test.sh -f sim/dnode/basic1.sim +./test.sh -f tsim/dnode/basic1.sim #======================b1-end=============== diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 3e8f87d665..ea1ad65a5b 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -113,6 +113,8 @@ elif [ $NODE -eq 7 ]; then NODE=7700 elif [ $NODE -eq 8 ]; then NODE=7800 +elif [ $NODE -eq 9 ]; then + NODE=7900 fi HOSTNAME=localhost diff --git a/tests/script/sh/dockerbuilder/addDnodeToCluster.sh b/tests/script/sh/dockerbuilder/addDnodeToCluster.sh new file mode 100755 index 0000000000..1886632350 --- /dev/null +++ b/tests/script/sh/dockerbuilder/addDnodeToCluster.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# deploy test cluster + +set -e +#set -x + +dnodeNumber=1 +serverPort=6030 + +while getopts "hn:p:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + p) + serverPort=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +for ((i=2; i<=${dnodeNumber}; i++)); do + taos -s "create dnode node${i} port ${serverPort};" ||: + echo "create dnode node${i} port ${serverPort};" +done + + + + diff --git a/tests/script/sh/dockerbuilder/addFqdnToHost.sh b/tests/script/sh/dockerbuilder/addFqdnToHost.sh new file mode 100755 index 0000000000..4d613d1dd0 --- /dev/null +++ b/tests/script/sh/dockerbuilder/addFqdnToHost.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# deploy test cluster + +set -e +#set -x + +dnodeNumber=1 +subnet="172.33.0.0/16" + +while getopts "hn:s:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + s) + subnet=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + echo " -s [ subnet] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +addFqdnToHosts() { + index=$1 + ipPrefix=$2 + let ipIndex=index+1 + echo "${ipPrefix}.${ipIndex} node${i}" >> /etc/hosts +} + +ipPrefix=${subnet%.*} +for ((i=1; i<=${dnodeNumber}; i++)); do + addFqdnToHosts ${i} ${ipPrefix} +done + + + + diff --git a/tests/script/sh/dockerbuilder/clientCfg/taos.cfg b/tests/script/sh/dockerbuilder/clientCfg/taos.cfg new file mode 100644 index 0000000000..491216d5f6 --- /dev/null +++ b/tests/script/sh/dockerbuilder/clientCfg/taos.cfg @@ -0,0 +1,4 @@ +debugFlag 131 +firstEp node1:6030 +asyncLog 0 +telemetryReporting 0 diff --git a/tests/script/sh/dockerbuilder/createComposeYml.sh b/tests/script/sh/dockerbuilder/createComposeYml.sh new file mode 100755 index 0000000000..494e79318b --- /dev/null +++ b/tests/script/sh/dockerbuilder/createComposeYml.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# +# create docker-compose.yml + +set -e +#set -x + +# set parameters by default value +composeYmlFile="./docker-compose.yml" +dnodeNumber=1 +subnet="172.33.0.0/16" + +while getopts "hn:f:s:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + f) + composeYmlFile=$(echo $OPTARG) + ;; + s) + subnet=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + echo " -f [ yml file] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "dnodeNumber=${dnodeNumber} composeYmlFile=${composeYmlFile}" + +createFirstSection() { + ymlFile=$1 + + echo "version: '3.7'" > ${ymlFile} + echo "" >> ${ymlFile} + echo "x-node: &x-node" >> ${ymlFile} + echo " build:" >> ${ymlFile} + echo " context: ." >> ${ymlFile} + echo " dockerfile: ./tdserver/Dockerfile" >> ${ymlFile} + echo " args:" >> ${ymlFile} + echo " - PACKAGE=TDengine-server-3.0.0.0-Linux-x64.tar.gz" >> ${ymlFile} + echo " - EXTRACTDIR=TDengine-server-3.0.0.0" >> ${ymlFile} + echo " image: 'tdengine:3.0.0.0'" >> ${ymlFile} + echo " container_name: 'node1'" >> ${ymlFile} + echo " privileged: true" >> ${ymlFile} + echo " cap_add:" >> ${ymlFile} + echo " - ALL" >> ${ymlFile} + echo " stdin_open: true" >> ${ymlFile} + echo " tty: true" >> ${ymlFile} + echo " environment:" >> ${ymlFile} + echo ' TZ: "Asia/Shanghai"' >> ${ymlFile} + echo ' command: >' >> ${ymlFile} + echo ' sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&' >> ${ymlFile} + echo ' echo $TZ > /etc/timezone &&' >> ${ymlFile} + echo ' exec sysctl -w kernel.core_pattern=/corefile/core-%e-%p"' >> ${ymlFile} + echo " restart: always" >> ${ymlFile} + echo " hostname: node1" >> ${ymlFile} + echo " command: taosd" >> ${ymlFile} + echo " deploy:" >> ${ymlFile} + echo " resources:" >> ${ymlFile} + echo " limits:" >> ${ymlFile} + echo ' cpus: "2.00"' >> ${ymlFile} + echo " memory: 4G" >> ${ymlFile} + echo " reservations:" >> ${ymlFile} + echo ' cpus: "1.00"' >> ${ymlFile} + echo " memory: 500M" >> ${ymlFile} + echo " volumes:" >> ${ymlFile} + echo " - /etc/localtime:/etc/localtime:ro" >> ${ymlFile} + echo ' - $PWD:/work' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/data:/var/lib/taos' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/log:/var/log/taos' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/cfg:/etc/taos' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/core:/corefile' >> ${ymlFile} + echo "" >> ${ymlFile} + echo "networks:" >> ${ymlFile} + echo " tdnet:" >> ${ymlFile} + echo " ipam:" >> ${ymlFile} + echo " driver: default" >> ${ymlFile} + echo " config:" >> ${ymlFile} + echo " - subnet: ${subnet}" >> ${ymlFile} + echo "" >> ${ymlFile} + echo "services:" >> ${ymlFile} +} + +createSingleDnodesCfg() { + ymlFile=$1 + index=$2 + + ipPrefix=${subnet%.*} + + let ipIndex=index+1 + + echo " node${index}:" >> ${ymlFile} + echo " <<: *x-node" >> ${ymlFile} + echo " container_name: 'node${index}'" >> ${ymlFile} + echo " hostname: node${index}" >> ${ymlFile} + echo " networks:" >> ${ymlFile} + echo " tdnet:" >> ${ymlFile} + echo " ipv4_address: ${ipPrefix}.${ipIndex}" >> ${ymlFile} + echo " volumes:" >> ${ymlFile} + echo " - /etc/localtime:/etc/localtime:ro" >> ${ymlFile} + echo " - \$PWD:/work" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/data:/var/lib/taos" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/log:/var/log/taos" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/cfg:/etc/taos" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/core:/corefile" >> ${ymlFile} + echo "" >> ${ymlFile} +} + +createDnodesOfDockerComposeYml() { + ymlFile=$1 + dnodeNumber=$2 + + for ((i=1; i<=${dnodeNumber}; i++)); do + createSingleDnodesCfg ${ymlFile} ${i} + done +} + +######################################################################################## +############################### main process ########################################## + +createFirstSection ${composeYmlFile} +createDnodesOfDockerComposeYml ${composeYmlFile} ${dnodeNumber} + +echo "====create docker-compose.yml end====" +echo " " + + diff --git a/tests/script/sh/dockerbuilder/createDnodeStorage.sh b/tests/script/sh/dockerbuilder/createDnodeStorage.sh new file mode 100755 index 0000000000..4e9f8e5121 --- /dev/null +++ b/tests/script/sh/dockerbuilder/createDnodeStorage.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# setup test environment + +set -e +#set -x + +# set parameters by default value +dataRootDir="/data/dockerbuilder/storage" +dnodeNumber=1 +firstEp="node1:6030" + +while getopts "hn:r:f:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + f) + firstEp=$(echo $OPTARG) + ;; + r) + dataRootDir=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + echo " -n [ dnode number] " + echo " -f [ first ep] " + echo " -r [ data root dir] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "dnodeNumber=${dnodeNumber} dataRootDir=${dataRootDir} firstEp=${firstEp}" + +createTaosCfg() { + cfgFile=$1/cfg/taos.cfg + firstEp=$2 + fqdn=$3 + + echo "debugFlag 131" > ${cfgFile} + echo "firstEp ${firstEp}" >> ${cfgFile} + #echo "dataDir ${dataDir}" >> ${cfgFile} + #echo "logDir ${logDir}" >> ${cfgFile} + echo "fqdn ${fqdn}" >> ${cfgFile} + + echo "supportVnodes 1024" >> ${cfgFile} + echo "asyncLog 0" >> ${cfgFile} + echo "telemetryReporting 0" >> ${cfgFile} +} + +createDnodesDataDir() { + if [ -d ${dataRootDir} ]; then + rm -rf ${dataRootDir}/* + else + echo "${dataRootDir} not exist" + exit 1 + fi + + dnodeNumber=$1 + firstEp=$2 + + serverPort=${startPort} + for ((i=1; i<=${dnodeNumber}; i++)); do + mkdir -p ${dataRootDir}/dnode${i}/cfg + mkdir -p ${dataRootDir}/dnode${i}/log + mkdir -p ${dataRootDir}/dnode${i}/data + mkdir -p ${dataRootDir}/dnode${i}/core + + createTaosCfg ${dataRootDir}/dnode${i} ${firstEp} node${i} + done +} + +######################################################################################## +############################### main process ########################################## + +## create director and taos.cfg for all dnode +createDnodesDataDir ${dnodeNumber} ${firstEp} + +echo "====create end====" +echo " " + + diff --git a/tests/script/sh/dockerbuilder/tdserver/Dockerfile b/tests/script/sh/dockerbuilder/tdserver/Dockerfile new file mode 100644 index 0000000000..7757aa68c7 --- /dev/null +++ b/tests/script/sh/dockerbuilder/tdserver/Dockerfile @@ -0,0 +1,42 @@ +FROM centos:8 AS builder + +ARG PACKAGE=TDengine-server-3.0.0.0-Linux-x64.tar.gz +ARG EXTRACTDIR=TDengine-server-3.0.0.0 + +WORKDIR /root + +COPY ${PACKAGE} . +COPY tmux.conf . +COPY addDnodeToCluster.sh . + +RUN tar -zxf ${PACKAGE} +RUN mv ${EXTRACTDIR}/* ./ + +FROM centos:8 + +WORKDIR /root + +RUN yum install -y glibc-langpack-en dmidecode gdb +RUN yum install -y tmux net-tools +RUN yum install -y sysstat +RUN yum install -y vim +RUN echo 'alias ll="ls -l --color=auto"' >> ~/.bashrc + +COPY --from=builder /root/taosd /usr/bin +COPY --from=builder /root/taos /usr/bin +COPY --from=builder /root/create_table /usr/bin +COPY --from=builder /root/tmux.conf /root/.tmux.conf +#COPY --from=builder /root/addDnodeToCluster.sh /root/addDnodeToCluster.sh + +#COPY --from=builder /root/cfg/taos.cfg /etc/taos/ +COPY --from=builder /root/lib/* /usr/lib/ + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LC_CTYPE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 + +EXPOSE 6030-6042/tcp 6060/tcp 6030-6039/udp + +# VOLUME [ "/var/lib/taos", "/var/log/taos", "/etc/taos" ] + +CMD [ "bash" ] diff --git a/tests/script/sh/dockerbuilder/tmux.conf b/tests/script/sh/dockerbuilder/tmux.conf new file mode 100644 index 0000000000..d1f1801f4a --- /dev/null +++ b/tests/script/sh/dockerbuilder/tmux.conf @@ -0,0 +1,115 @@ +unbind C-b +# remap prefix to Control + z +set -g prefix C-a +# bind 'C-z C-z' to type 'C-z' +bind C-a send-prefix + + +# 0 is too far from ` ;) +set -g base-index 1 + +# Automatically set window title +set-window-option -g automatic-rename on +set-option -g set-titles on + +#set -g default-terminal screen-256color +set -g status-keys vi +set -g history-limit 10000 + +setw -g mode-keys vi +#setw -g mode-mouse on +setw -g monitor-activity on +set -g mouse on + +bind-key v split-window -h -c "#{pane_current_path}" +bind-key s split-window -v -c "#{pane_current_path}" +bind-key c new-window -c "#{pane_current_path}" + +bind-key -n C-S-Left swap-window -t -1 +bind-key -n C-S-Right swap-window -t +1 + +# bind-key J resize-pane -D 5 +# bind-key K resize-pane -U 5 +# bind-key H resize-pane -L 5 +# bind-key L resize-pane -R 5 + +# bind-key M-j resize-pane -D +# bind-key M-k resize-pane -U +# bind-key M-h resize-pane -L +# bind-key M-l resize-pane -R + +# Vim style pane selection +bind h select-pane -L +bind j select-pane -D +bind k select-pane -U +bind l select-pane -R +bind K kill-pane -a + +# Use Alt-vim keys without prefix key to switch panes +bind -n M-h select-pane -L +bind -n M-j select-pane -D +bind -n M-k select-pane -U +bind -n M-l select-pane -R +# bind -n M-h select-pane -L \; resize-pane -Z +# bind -n M-j select-pane -D \; resize-pane -Z +# bind -n M-k select-pane -U \; resize-pane -Z +# bind -n M-l select-pane -R \; resize-pane -Z + +# # Use Alt-arrow keys without prefix key to switch panes +bind -n M-Left select-pane -L +bind -n M-Right select-pane -R +bind -n M-Up select-pane -U +bind -n M-Down select-pane -D + +# Shift arrow to switch windows +bind -n C-Left previous-window +bind -n C-Right next-window + +bind -n M-n previous-window +bind -n M-p nejt-window + +# Map Alt-z to zoom in and out pane +bind -n M-z resize-pane -Z +bind -n M-q resize-pane -Z + +# # Use v to trigger selection +bind-key -T copy-mode-vi v send-keys -X begin-selection +# +# # Use y to yank current selection +bind-key -T copy-mode-vi y send-keys -X copy-selection-and-cancel + +# No delay for escape key press +set -sg escape-time 0 + +# THEME +set -g status-style bg=black,fg=white +# set -g status-bg black +# set -g status-fg white +set -g window-status-current-style bg=green,fg=black,bold +set -g pane-active-border-style fg=green,bg=black +set -g status-interval 60 +set -g status-left-length 30 +set -g status-left '#[fg=green](#S) #(whoami)' +set -g status-right '#[fg=yellow]#(cut -d " " -f 1-3 /proc/loadavg)#[default] #[fg=white]%H:%M#[default]' + +# synchronouse windows +# setw synchronize-panes + +# pane movement +bind-key b command-prompt -p "join pane from:" "join-pane -s ':%%'" +bind-key a command-prompt -p "send pane to:" "join-pane -t ':%%'" +# # for vim +# # Smart pane switching with awareness of Vim splits. +# # See: https://github.com/christoomey/vim-tmux-navigator +# is_vim="ps -o state= -o comm= -t '#{pane_tty}' \ +# | grep -iqE '^[^TXZ ]+ +(\\S+\\/)?g?(view|n?vim?x?)(diff)?$'" +# bind-key -n C-h if-shell "$is_vim" "send-keys C-h" "select-pane -L" +# bind-key -n C-j if-shell "$is_vim" "send-keys C-j" "select-pane -D" +# bind-key -n C-k if-shell "$is_vim" "send-keys C-k" "select-pane -U" +# bind-key -n C-l if-shell "$is_vim" "send-keys C-l" "select-pane -R" +# bind-key -n C-\ if-shell "$is_vim" "send-keys C-\\" "select-pane -l" +# bind-key -T copy-mode-vi C-h select-pane -L +# bind-key -T copy-mode-vi C-j select-pane -D +# bind-key -T copy-mode-vi C-k select-pane -U +# bind-key -T copy-mode-vi C-l select-pane -R +# bind-key -T copy-mode-vi C-\ select-pane -l diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh index beb324321d..503f3ccae2 100755 --- a/tests/script/sh/massiveTable/deployCluster.sh +++ b/tests/script/sh/massiveTable/deployCluster.sh @@ -5,10 +5,12 @@ set -e #set -x +fqdn=`hostname` + masterDnode=slave dataRootDir="/data" -firstEp="trd02:7000" -startPort=7000 +firstEp="${fqdn}:6030" +startPort=6030 dnodeNumber=1 updateSrc=no @@ -52,23 +54,17 @@ done curr_dir=$(readlink -f "$(dirname "$0")") echo $curr_dir -${curr_dir}/cleanCluster.sh -r "/data" -#${curr_dir}/cleanCluster.sh -r "/data2" +${curr_dir}/cleanCluster.sh -r ${dataRootDir} if [[ "${updateSrc}" == "yes" ]]; then ${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" fi -${curr_dir}/setupDnodes.sh -r "/data" -n ${dnodeNumber} -f ${firstEp} -p 7000 -#${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000 +${curr_dir}/setupDnodes.sh -r ${dataRootDir} -n ${dnodeNumber} -f ${firstEp} -p ${startPort} if [[ "${masterDnode}" == "master" ]]; then - # create all dnode into cluster - #taos -s "create dnode trd02 port 8000;" - taos -s "create dnode trd03 port 7000;" - #taos -s "create dnode trd03 port 8000;" - taos -s "create dnode trd04 port 7000;" - #taos -s "create dnode trd04 port 8000;" + taos -s "create dnode trd03 port 6030;" + taos -s "create dnode trd04 port 6030;" fi diff --git a/tests/script/sh/massiveTable/setupDnodes.sh b/tests/script/sh/massiveTable/setupDnodes.sh index 37fdbaf784..22c325cd89 100755 --- a/tests/script/sh/massiveTable/setupDnodes.sh +++ b/tests/script/sh/massiveTable/setupDnodes.sh @@ -13,10 +13,12 @@ set -e # -r [ dnode root dir] # set parameters by default value +fqdn=`hostname` + enviMode=new dataRootDir="/data" -firstEp="localhost:7000" -startPort=7000 +firstEp="${fqdn}:6030" +startPort=6030 dnodeNumber=1 diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index 28675be808..3b43656a41 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -5,7 +5,41 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 2 -system sh/cfg.sh -n dnode2 -c walLevel -v 2 -system sh/cfg.sh -n dnode3 -c walLevel -v 2 -system sh/cfg.sh -n dnode4 -c walLevel -v 2 +return + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/deploy.sh -n dnode6 -i 6 +system sh/deploy.sh -n dnode7 -i 7 +system sh/deploy.sh -n dnode8 -i 8 +system sh/deploy.sh -n dnode9 -i 9 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode6 -s start +system sh/exec.sh -n dnode7 -s start +system sh/exec.sh -n dnode8 -s start +system sh/exec.sh -n dnode9 -s start + +sleep 2000 + +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 +sql create dnode $hostname port 7600 +sql create dnode $hostname port 7700 +sql create dnode $hostname port 7800 +sql create dnode $hostname port 7900 + +sql show dnodes; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 +print $data40 $data41 \ No newline at end of file diff --git a/tests/script/sim/db/basic1.sim b/tests/script/tsim/db/basic1.sim similarity index 100% rename from tests/script/sim/db/basic1.sim rename to tests/script/tsim/db/basic1.sim diff --git a/tests/script/sim/db/basic6.sim b/tests/script/tsim/db/basic6.sim similarity index 100% rename from tests/script/sim/db/basic6.sim rename to tests/script/tsim/db/basic6.sim diff --git a/tests/script/sim/db/error1.sim b/tests/script/tsim/db/error1.sim similarity index 100% rename from tests/script/sim/db/error1.sim rename to tests/script/tsim/db/error1.sim diff --git a/tests/script/sim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim similarity index 100% rename from tests/script/sim/dnode/basic1.sim rename to tests/script/tsim/dnode/basic1.sim diff --git a/tests/script/sim/table/basic1.sim b/tests/script/tsim/table/basic1.sim similarity index 100% rename from tests/script/sim/table/basic1.sim rename to tests/script/tsim/table/basic1.sim diff --git a/tests/script/sim/user/basic1.sim b/tests/script/tsim/user/basic1.sim similarity index 100% rename from tests/script/sim/user/basic1.sim rename to tests/script/tsim/user/basic1.sim diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index fddf3e7f2a..f1b6b6b502 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -31,6 +31,7 @@ int32_t createTable = 1; int32_t insertData = 0; int32_t batchNumOfTbl = 100; int32_t batchNumOfRow = 1; +int32_t totalRowsOfPerTbl = 1; int32_t numOfVgroups = 2; int32_t showTablesFlag = 0; int32_t queryFlag = 0; @@ -106,14 +107,14 @@ void printCreateProgress(SThreadInfo *pInfo, int64_t t) { totalTables, seconds, speed); } -void printInsertProgress(SThreadInfo *pInfo, int64_t t) { +void printInsertProgress(SThreadInfo *pInfo, int64_t insertTotalRows) { int64_t endMs = taosGetTimestampMs(); - int64_t totalTables = t - pInfo->tableBeginIndex; + //int64_t totalTables = t - pInfo->tableBeginIndex; float seconds = (endMs - pInfo->startMs) / 1000.0; - float speed = totalTables / seconds; + float speed = insertTotalRows / seconds; pInfo->insertDataSpeed = speed; pPrint("thread:%d, %" PRId64 " rows inserted, time:%.2f sec, speed:%.1f rows/second, ", pInfo->threadIndex, - totalTables, seconds, speed); + insertTotalRows, seconds, speed); } static int64_t getResult(TAOS_RES *tres) { @@ -181,14 +182,13 @@ void *threadFunc(void *param) { exit(1); } - pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, - pInfo->tableEndIndex); + //pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, + // pInfo->tableEndIndex); pInfo->tableBeginIndex += startOffset; pInfo->tableEndIndex += startOffset; - pPrint("====after thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, - pInfo->tableEndIndex); + pPrint("====thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); sprintf(qstr, "use %s", pInfo->dbName); TAOS_RES *pRes = taos_query(con, qstr); @@ -237,51 +237,50 @@ void *threadFunc(void *param) { } if (insertData) { + int64_t insertTotalRows = 0; int64_t curMs = 0; int64_t beginMs = taosGetTimestampMs(); pInfo->startMs = beginMs; int64_t t = pInfo->tableBeginIndex; - for (; t <= pInfo->tableEndIndex;) { - // int64_t batch = (pInfo->tableEndIndex - t); - // batch = MIN(batch, batchNum); - - int32_t len = sprintf(qstr, "insert into "); - - for (int32_t i = 0; i < batchNumOfTbl;) { - int64_t ts = startTimestamp; + for (; t <= pInfo->tableEndIndex; t++) { + //printf("table name: %"PRId64"\n", t); + int64_t ts = startTimestamp; + for (int32_t i = 0; i < totalRowsOfPerTbl;) { + int32_t len = sprintf(qstr, "insert into "); len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t); for (int32_t j = 0; j < batchNumOfRow; j++) { len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); + i++; + insertTotalRows++; + if (i >= totalRowsOfPerTbl) { + break; + } } - t++; - i++; - if (t > pInfo->tableEndIndex) { - break; + #if 1 + int64_t startTs = taosGetTimestampUs(); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { + pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code)); } + taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; + + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; + // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); + printInsertProgress(pInfo, insertTotalRows); + } + #endif } - - int64_t startTs = taosGetTimestampUs(); - TAOS_RES *pRes = taos_query(con, qstr); - code = taos_errno(pRes); - if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { - pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code)); - } - taos_free_result(pRes); - int64_t endTs = taosGetTimestampUs(); - int64_t delay = endTs - startTs; - // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); - if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; - if (delay < pInfo->minDelay) pInfo->minDelay = delay; - - curMs = taosGetTimestampMs(); - if (curMs - beginMs > 10000) { - beginMs = curMs; - // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); - printInsertProgress(pInfo, t); - } - } - printInsertProgress(pInfo, t); + } + printInsertProgress(pInfo, insertTotalRows); } taos_close(con); @@ -319,6 +318,8 @@ void printHelp() { printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag); printf("%s%s\n", indent, "-l"); printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", totalRowsOfPerTbl); exit(EXIT_SUCCESS); } @@ -350,6 +351,8 @@ void parseArgument(int32_t argc, char *argv[]) { batchNumOfTbl = atoi(argv[++i]); } else if (strcmp(argv[i], "-l") == 0) { batchNumOfRow = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + totalRowsOfPerTbl = atoi(argv[++i]); } else if (strcmp(argv[i], "-w") == 0) { showTablesFlag = atoi(argv[++i]); } else if (strcmp(argv[i], "-q") == 0) { @@ -370,6 +373,7 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s insertData:%d %s", GREEN, insertData, NC); pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC); pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC); + pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, totalRowsOfPerTbl, NC); pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC); diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/backup/tnettest.c index 7b5dbd2405..be8714387f 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/backup/tnettest.c @@ -319,7 +319,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) { char user[] = "nettestinternal"; char pass[] = "nettestinternal"; - taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); + taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; @@ -344,7 +344,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p SRpcMsg rspMsg; void * pRpcConn; - char secretEncrypt[32] = {0}; + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; pRpcConn = taosNetInitRpc(secretEncrypt, spi); if (NULL == pRpcConn) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1ad61ee2b0..c283423fbf 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -751,56 +751,56 @@ static int calcColWidth(TAOS_FIELD *field, int precision) { switch (field->type) { case TSDB_DATA_TYPE_BOOL: - return MAX(5, width); // 'false' + return TMAX(5, width); // 'false' case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: - return MAX(4, width); // '-127' + return TMAX(4, width); // '-127' case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: - return MAX(6, width); // '-32767' + return TMAX(6, width); // '-32767' case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: - return MAX(11, width); // '-2147483648' + return TMAX(11, width); // '-2147483648' case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: - return MAX(21, width); // '-9223372036854775807' + return TMAX(21, width); // '-9223372036854775807' case TSDB_DATA_TYPE_FLOAT: - return MAX(20, width); + return TMAX(20, width); case TSDB_DATA_TYPE_DOUBLE: - return MAX(25, width); + return TMAX(25, width); case TSDB_DATA_TYPE_BINARY: if (field->bytes > tsMaxBinaryDisplayWidth) { - return MAX(tsMaxBinaryDisplayWidth, width); + return TMAX(tsMaxBinaryDisplayWidth, width); } else { - return MAX(field->bytes, width); + return TMAX(field->bytes, width); } case TSDB_DATA_TYPE_NCHAR: { int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; if (bytes > tsMaxBinaryDisplayWidth) { - return MAX(tsMaxBinaryDisplayWidth, width); + return TMAX(tsMaxBinaryDisplayWidth, width); } else { - return MAX(bytes, width); + return TMAX(bytes, width); } } case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { - return MAX(14, width); + return TMAX(14, width); } if (precision == TSDB_TIME_PRECISION_NANO) { - return MAX(29, width); + return TMAX(29, width); } else if (precision == TSDB_TIME_PRECISION_MICRO) { - return MAX(26, width); // '2020-01-01 00:00:00.000000' + return TMAX(26, width); // '2020-01-01 00:00:00.000000' } else { - return MAX(23, width); // '2020-01-01 00:00:00.000' + return TMAX(23, width); // '2020-01-01 00:00:00.000' } default: