diff --git a/Jenkinsfile2 b/Jenkinsfile2 index cbf663cdcf..7a6f793399 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -117,27 +117,29 @@ def pre_test(){ def pre_test_win(){ bat ''' hostname + ipconfig + set date /t time /t - taskkill /f /t /im python.exe - taskkill /f /t /im bash.exe rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug - exit 0 ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git reset --hard git fetch || git fetch + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git reset --hard git fetch || git fetch - git checkout -f ''' script { if (env.CHANGE_TARGET == 'master') { bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout master + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout master ''' @@ -145,6 +147,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout 2.0 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout 2.0 ''' @@ -152,6 +156,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout 3.0 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout 3.0 ''' @@ -159,6 +165,8 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout develop + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout develop ''' @@ -169,30 +177,52 @@ def pre_test_win(){ bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git pull - git log -5 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git pull - git fetch origin +refs/pull/${CHANGE_ID}/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git fetch origin +refs/pull/%CHANGE_ID%/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git checkout -qf FETCH_HEAD - git log -5 ''' } else if (env.CHANGE_URL =~ /\/TDinternal\//) { bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git pull - git fetch origin +refs/pull/${CHANGE_ID}/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git fetch origin +refs/pull/%CHANGE_ID%/merge + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal git checkout -qf FETCH_HEAD - git log -5 + ''' + bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git pull - git log -5 ''' } else { - sh ''' - echo "unmatched reposiotry ${CHANGE_URL}" + bat ''' + echo "unmatched reposiotry %CHANGE_URL%" ''' } } + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + git branch + git log -5 + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + git branch + git log -5 + ''' bat ''' cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git submodule update --init --recursive @@ -205,10 +235,15 @@ def pre_test_build_win() { cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal mkdir debug cd debug + time /t call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 set CL=/MP8 - cmake .. -G "NMake Makefiles JOM" - jom -j 4 || exit 8 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" + time /t + cmake .. -G "NMake Makefiles JOM" || exit 7 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" + time /t + jom -j 6 || exit 8 time /t ''' return 1 @@ -226,6 +261,13 @@ pipeline { stages { stage('run test') { parallel { + stage('windows test') { + agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "} + steps { + pre_test_win() + pre_test_build_win() + } + } stage('linux test') { agent{label " slave3_0 || slave15 || slave16 || slave17 "} options { skipDefaultCheckout() } diff --git a/cmake/cmake.install b/cmake/cmake.install index f51c6566fa..b2421fac25 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -5,22 +5,27 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) SET(CMAKE_INSTALL_PREFIX C:/TDengine) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) - INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) + # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) - INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taos.h DESTINATION include) - INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taoserror.h DESTINATION include) + INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include) + INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .) IF (TD_MVN_INSTALLED) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc) ENDIF () + SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat") + INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})") ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") diff --git a/example/src/tmq.c b/example/src/tmq.c index b79d21d051..1abce3f188 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -239,7 +239,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { msg_process(tmqmessage); taos_free_result(tmqmessage); - tmq_commit(tmq, NULL, 1); + tmq_commit_async(tmq, NULL, tmq_commit_cb_print, NULL); /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } } diff --git a/include/client/taos.h b/include/client/taos.h index 0194357841..0b8c67aa79 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -232,11 +232,11 @@ DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); -DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); -DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets); +DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); + #if 0 -DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); +DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7a60542313..f7562a4b9b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1480,6 +1480,7 @@ typedef struct { typedef struct { int64_t consumerId; char cgroup[TSDB_CGROUP_LEN]; + char clientId[256]; SArray* topicNames; // SArray } SCMSubscribeReq; @@ -1487,6 +1488,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->cgroup); + tlen += taosEncodeString(buf, pReq->clientId); int32_t topicNum = taosArrayGetSize(pReq->topicNames); tlen += taosEncodeFixedI32(buf, topicNum); @@ -1500,6 +1502,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->cgroup); + buf = taosDecodeStringTo(buf, pReq->clientId); int32_t topicNum; buf = taosDecodeFixedI32(buf, &topicNum); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 04a24c4f32..f9d8fc0de1 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -59,6 +59,11 @@ typedef struct SMetaData { SArray *pQnodeList; // qnode list, SArray } SMetaData; +typedef struct STbSVersion { + char* tbFName; + int32_t sver; +} STbSVersion; + typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; @@ -165,6 +170,8 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); */ int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName); +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables); + /** * Force refresh a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 711db65e97..daf008108b 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -220,23 +220,23 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ } \ } while (0) -#define qDebug(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebug(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qTrace(...) \ - do { \ - if (qDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ - } \ +#define qTrace(...) \ + do { \ + if (qDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define qDebugL(...) \ - do { \ - if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ - } \ +#define qDebugL(...) \ + do { \ + if (qDebugFlag & DEBUG_DEBUG) { \ + taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ + } \ } while (0) #define QRY_ERR_RET(c) \ diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index b3f35025d1..dcd058a293 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, bool needRes, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. diff --git a/include/os/osString.h b/include/os/osString.h index 5f65f97bec..1b518f9b81 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -38,6 +38,13 @@ typedef int32_t TdUcs4; #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID #define wchar_t WCHAR_T_TYPE_TAOS_FORBID #define strcasestr STR_CASE_STR_FORBID + #define strtoll STR_TO_LL_FUNC_TAOS_FORBID + #define strtoull STR_TO_ULL_FUNC_TAOS_FORBID + #define strtol STR_TO_L_FUNC_TAOS_FORBID + #define strtoul STR_TO_UL_FUNC_TAOS_FORBID + #define strtod STR_TO_LD_FUNC_TAOS_FORBID + #define strtold STR_TO_D_FUNC_TAOS_FORBID + #define strtof STR_TO_F_FUNC_TAOS_FORBID #endif #ifdef WINDOWS @@ -72,6 +79,17 @@ int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size); char *taosStrCaseStr(const char *str, const char *pattern); +int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix); +uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix); +int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix); +uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix); +int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix); +uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix); +int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix); +uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix); +double taosStr2Double(const char *str, char** pEnd); +float taosStr2Float(const char *str, char** pEnd); + #ifdef __cplusplus } #endif diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index eeae1b47da..10d3dcdbaa 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -56,10 +56,10 @@ typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutS typedef struct SSkipList { uint32_t seed; + uint16_t len; __compar_fn_t comparFn; __sl_key_fn_t keyFn; TdThreadRwlock *lock; - uint16_t len; uint8_t maxLevel; uint8_t flags; uint8_t type; // static info above diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat new file mode 100644 index 0000000000..64f30b8465 --- /dev/null +++ b/packaging/tools/make_install.bat @@ -0,0 +1,6 @@ +@echo off +goto %1 +:needAdmin +mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&goto :eof +:hasAdmin +cp -f C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 \ No newline at end of file diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c82cce9af3..30997def74 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -83,6 +83,15 @@ void closeTransporter(STscObj *pTscObj) { rpcClose(pTscObj->pAppInfo->pTransporter); } +static bool clientRpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; + } +} + + // TODO refactor void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; @@ -91,6 +100,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; + rpcInit.rfp = clientRpcRfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 23787651b2..7693d26d3f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -291,7 +291,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, NULL != pRes, &res); + pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); @@ -310,9 +310,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } } - if (pRes) { - *pRes = res.res; - } + *pRes = res.res; pRequest->code = res.code; terrno = res.code; @@ -324,7 +322,60 @@ int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList); } +int32_t validateSversion(SRequestObj* pRequest, void* res) { + SArray* pArray = NULL; + int32_t code = 0; + + if (TDMT_VND_SUBMIT == pRequest->type) { + SSubmitRsp* pRsp = (SSubmitRsp*)res; + if (pRsp->nBlocks <= 0) { + return TSDB_CODE_SUCCESS; + } + + pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); + if (NULL == pArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pRsp->nBlocks; ++i) { + SSubmitBlkRsp *blk = pRsp->pBlocks + i; + STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; + taosArrayPush(pArray, &tbSver); + } + } else if (TDMT_VND_QUERY == pRequest->type) { + + } + + SCatalog* pCatalog = NULL; + CHECK_CODE_GOTO(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog), _return); + + SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + code = catalogChkTbMetaVersion(pCatalog, pRequest->pTscObj->pAppInfo->pTransporter, &epset, pArray); + +_return: + + taosArrayDestroy(pArray); + + return code; +} + +void freeRequestRes(SRequestObj* pRequest, void* res) { + if (NULL == res) { + return; + } + + if (TDMT_VND_SUBMIT == pRequest->type) { + tFreeSSubmitRsp((SSubmitRsp*)res); + } else if (TDMT_VND_QUERY == pRequest->type) { + + } +} + SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { + void* pRes = NULL; + if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: @@ -337,7 +388,10 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); if (TSDB_CODE_SUCCESS == code) { - code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, res); + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, &pRes); + if (NULL != pRes) { + code = validateSversion(pRequest, pRes); + } } taosArrayDestroy(pNodeList); break; @@ -356,6 +410,12 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; + freeRequestRes(pRequest, pRes); + pRes = NULL; + } + + if (res) { + *res = pRes; } return pRequest; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f6f8db8590..d5377c99a6 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -580,7 +580,7 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){ const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; - double result = strtod(pVal, &endptr); + double result = taosStr2Double(pVal, &endptr); if(pVal == endptr){ smlBuildInvalidDataMsg(msg, "invalid data", pVal); return false; @@ -714,7 +714,7 @@ static bool smlIsNchar(const char *pVal, uint16_t len) { static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { char *endPtr = NULL; - int64_t tsInt64 = strtoll(value, &endPtr, 10); + int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); if(value + len != endPtr){ return -1; } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index e4e5abe148..9fe7645e2b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -263,7 +263,7 @@ static const SSysDbTableSchema topicSchema[] = { static const SSysDbTableSchema consumerSchema[] = { {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "consumer_group", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topics", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4e97ebbe47..278fc2c193 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4087,10 +4087,8 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1; if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1; - if (pBlock->hashMeta) { - if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; - if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; - } + if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; + if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1; if (tEncodeI64v(pEncoder, pBlock->sver) < 0) return -1; @@ -4104,12 +4102,10 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1; - if (pBlock->hashMeta) { - if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); - if (NULL == pBlock->tblFName) return -1; - if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; - } + if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; + pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); + if (NULL == pBlock->tblFName) return -1; + if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; diff --git a/source/common/src/tname.c b/source/common/src/tname.c index fa9b6e1e63..0764ea84b9 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -250,7 +250,7 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return -1; } - dst->acctId = strtoll(str, NULL, 10); + dst->acctId = taosStr2Int32(str, NULL, 10); } if ((type & T_NAME_DB) == T_NAME_DB) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d1516403c1..22bdd960ea 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1063,7 +1063,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { if (isAllRowsNone(pCol)) { - pVal->valType = TD_VTYPE_NULL; + pVal->valType = TD_VTYPE_NONE; #ifdef TD_SUPPORT_READ2 pVal->val = (void *)getNullValue(pCol->type); #else diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 62a7179626..69ba964187 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -590,7 +590,7 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura char* endPtr = NULL; /* get the basic numeric value */ - int64_t timestamp = strtoll(token, &endPtr, 10); + int64_t timestamp = taosStr2Int64(token, &endPtr, 10); if (errno != 0) { return -1; } @@ -608,7 +608,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati errno = 0; /* get the basic numeric value */ - *duration = strtoll(token, NULL, 10); + *duration = taosStr2Int64(token, NULL, 10); if (errno != 0) { return -1; } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index e44450fe6e..7b0bef4918 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -39,7 +39,7 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) { errno = 0; char *endPtr = NULL; - *value = strtoll(z, &endPtr, base); + *value = taosStr2Int64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; return -1; @@ -58,7 +58,7 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) { return -1; } - *value = strtoull(z, &endPtr, base); + *value = taosStr2UInt64(z, &endPtr, base); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { errno = 0; return -1; @@ -434,7 +434,7 @@ static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *val // return -1; // } // - // *value = strtod(pStr, NULL); + // *value = taosStr2Double(pStr, NULL); return 0; } @@ -911,7 +911,7 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { case TSDB_DATA_TYPE_DOUBLE: { if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { errno = 0; - double v = strtod(pVariant->pz, NULL); + double v = taosStr2Double(pVariant->pz, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { taosMemoryFree(pVariant->pz); return -1; diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index 14c6250b2a..e019924268 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -38,9 +38,9 @@ static void bmSendErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7894eb5acb..38e71754b6 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -116,8 +116,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; dmGetMnodeLoads(pMgmt, &minfo); - if (minfo.isMnode && minfo.load.syncState != TAOS_SYNC_STATE_LEADER && - minfo.load.syncState != TAOS_SYNC_STATE_CANDIDATE) { + if (minfo.isMnode && minfo.load.syncState == TAOS_SYNC_STATE_ERROR) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); return; @@ -127,7 +126,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { dmGetVnodeLoads(pMgmt, &vinfo); for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); - if (pLoad->syncState != TAOS_SYNC_STATE_LEADER && pLoad->syncState != TAOS_SYNC_STATE_FOLLOWER) { + if (pLoad->syncState == TAOS_SYNC_STATE_ERROR) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pLoad->vgId, syncStr(pLoad->syncState)); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index be8ddf1430..599fa07e1c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -153,9 +153,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { if (code != 0 && terrno != 0) code = terrno; SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; rpcSendResponse(&rsp); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 98e234af3c..9b9960cf3a 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -19,9 +19,9 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index c94cf527c7..fcfc4f4cee 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -19,9 +19,9 @@ static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index 757f609731..d486707c5e 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -433,7 +433,7 @@ void dmCloseProcRpcHandles(SProc *proc) { SRpcHandleInfo *pInfo = taosHashIterate(proc->hash, NULL); while (pInfo != NULL) { dError("node:%s, the child process dies and send an offline rsp to handle:%p", proc->name, pInfo->handle); - SRpcMsg rpcMsg = {.info = *pInfo, .code = TSDB_CODE_NODE_OFFLINE}; + SRpcMsg rpcMsg = {.code = TSDB_CODE_NODE_OFFLINE, .info = *pInfo}; rpcSendResponse(&rpcMsg); pInfo = taosHashIterate(proc->hash, pInfo); } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ec9139836a..a04417736a 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -463,7 +463,7 @@ typedef struct { typedef struct { int64_t consumerId; char cgroup[TSDB_CGROUP_LEN]; - char appId[TSDB_CGROUP_LEN]; + char clientId[256]; int8_t updateType; // used only for update int32_t epoch; int32_t status; diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 1e95859157..9bf7b6eb89 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -37,7 +37,7 @@ int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 274876567e..4fe7d8a399 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -427,6 +427,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerOld = mndAcquireConsumer(pMnode, consumerId); if (pConsumerOld == NULL) { pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); + tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256); pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; pConsumerNew->rebNewTopics = newSub; subscribe.topicNames = NULL; @@ -848,11 +849,11 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false); // app id - char appId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(varDataVal(appId), pConsumer->appId, TSDB_CGROUP_LEN); - varDataSetLen(appId, strlen(varDataVal(appId))); + char clientId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(clientId), pConsumer->clientId, TSDB_CGROUP_LEN); + varDataSetLen(clientId, strlen(varDataVal(clientId))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)appId, false); + colDataAppend(pColInfo, numOfRows, (const char *)clientId, false); // status char status[20 + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c3e5195ad2..0bf7b240b2 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -261,8 +261,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool isRedo) { +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid) { STransAction action = {0}; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); @@ -279,48 +278,29 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - if (isRedo) { - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } else { - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; } return 0; } -static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, - bool isRedo) { +static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { STransAction action = {0}; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + void *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_ALTER_VNODE; - if (isRedo) { - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - } else { - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; } return 0; @@ -487,7 +467,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid) != 0) { return -1; } } @@ -726,11 +706,8 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { - return -1; - } + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup) != 0) { + return -1; } } else { SVgObj newVgroup = {0}; @@ -744,9 +721,9 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return -1; } newVgroup.replica = pDb->cfg.replications; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1; } else { mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); @@ -757,7 +734,7 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj return -1; } newVgroup.replica = pDb->cfg.replications; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index c853794e86..c82472eec0 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -206,7 +206,7 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { int32_t totalVgNum = pOutput->pSub->vgNum; - mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); + mInfo("mq rebalance: subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -231,6 +231,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from consumer %ld", pVgEp->vgId, consumerId); } taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); // put into removed @@ -250,6 +251,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from unassigned", pVgEp->vgId); } } @@ -263,6 +265,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR minVgCnt = totalVgNum / afterRebConsumerNum; imbConsumerNum = totalVgNum % afterRebConsumerNum; } + mInfo("mq rebalance: %d consumer after rebalance, at least %d vg each, %d consumer has more vg", afterRebConsumerNum, + minVgCnt, imbConsumerNum); // 4. first scan: remove consumer more than wanted, put to remove hash int32_t imbCnt = 0; @@ -290,6 +294,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId); } imbCnt++; } @@ -303,6 +308,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); + mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId); } } } @@ -319,6 +325,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosArrayPush(pOutput->newConsumers, &consumerId); + mInfo("mq rebalance: add new consumer %ld", consumerId); } } @@ -343,6 +350,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); + mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } @@ -360,6 +368,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); + mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } else { // if all consumer is removed, put all vg into unassigned @@ -372,6 +381,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ASSERT(pRebOutput->newConsumerId == -1); taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->rebVgs, pRebOutput); + mInfo("mq rebalance: unassign vg %d (second scan)", pRebOutput->pVgEp->vgId); } } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 3f559cb6c0..c6eebb5c5d 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -492,8 +492,8 @@ static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTo } static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SMnode *pMnode = pReq->info.node; + /*SSdb *pSdb = pMnode->pSdb;*/ SMDropTopicReq dropReq = {0}; if (tDeserializeSMDropTopicReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { @@ -502,16 +502,16 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } SMqTopicObj *pTopic = mndAcquireTopic(pMnode, dropReq.name); - // if (pTopic == NULL) { - // if (dropReq.igNotExists) { - // mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); - // return 0; - // } else { - // terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; - // mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); - // return -1; - // } - // } + if (pTopic == NULL) { + if (dropReq.igNotExists) { + mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); + return 0; + } else { + terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; + mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + return -1; + } + } if (pTopic->refConsumerCnt != 0) { mndReleaseTopic(pMnode, pTopic); @@ -528,12 +528,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); -#if 1 if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); return -1; } -#endif if (mndDropSubByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c9b68f27e7..35ecaa748e 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -842,13 +842,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, - pTrans->rpcInfo.ahandle); + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code, pTrans->stage, pTrans->rpcInfo.ahandle); SRpcMsg rspMsg = { - .info = pTrans->rpcInfo, .code = code, .pCont = rpcCont, .contLen = pTrans->rpcRspLen, + .info = pTrans->rpcInfo, }; tmsgSendRsp(&rspMsg); pTrans->rpcInfo.handle = NULL; @@ -986,7 +985,8 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); if (tmsgSendReq(&pAction->epSet, &rpcMsg) == 0) { - mDebug("trans:%d, action:%d is sent", pTrans->id, action); + mDebug("trans:%d, action:%d is sent to %s:%u", pTrans->id, action, pAction->epSet.eps[pAction->epSet.inUse].fqdn, + pAction->epSet.eps[pAction->epSet.inUse].port); pAction->msgSent = 1; pAction->msgReceived = 0; pAction->errCode = 0; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index f8c717edf6..62021c6a7e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -256,7 +256,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return pReq; } -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SAlterVnodeReq alterReq = {0}; alterReq.vgVersion = pVgroup->version; alterReq.buffer = pDb->cfg.buffer; @@ -285,16 +285,14 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgO pReplica->port = pVgidDnode->port; memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); - - if (pDnode->id == pVgid->dnodeId) { - alterReq.selfIndex = v; - } } +#if 0 if (alterReq.selfIndex == -1) { terrno = TSDB_CODE_MND_APP_ERROR; return NULL; } +#endif int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); if (contLen < 0) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 5326376bda..8c805dd8c7 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -489,7 +489,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; } - if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_CANDIDATE) { + if (pVgid->role != TAOS_SYNC_STATE_ERROR) { pClusterInfo->vnodes_alive++; } pClusterInfo->vnodes_total++; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 38dedee5a2..d0420e1b84 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -179,6 +179,7 @@ struct STQ { SHashObj* pStreamTasks; SVnode* pVnode; SWal* pWal; + // TDB* pTdb; }; typedef struct { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 6a5ad63822..b1760e6dae 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -104,6 +104,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb); +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 80aa350e4d..30de2dcfa0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -32,6 +32,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { pTq->path = strdup(path); pTq->pVnode = pVnode; pTq->pWal = pWal; + /*if (tdbOpen(path, 4096, 1, &pTq->pTdb) < 0) {*/ + /*ASSERT(0);*/ + /*}*/ #if 0 pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, @@ -108,6 +111,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList) { pIter = taosHashIterate(pTq->execs, pIter); if (pIter == NULL) break; pExec = (STqExec*)pIter; + if (pExec->subType == TOPIC_SUB_TYPE__DB) continue; for (int32_t i = 0; i < 5; i++) { int32_t code = qUpdateQualifiedTableId(pExec->task[i], tbUidList, true); ASSERT(code == 0); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 2fdbfdd206..d8426db127 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -320,6 +320,8 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } + strcat(pRsp->tblFName, mr.me.name); + if (mr.me.type == TSDB_NORMAL_TABLE) { sverNew = mr.me.ntbEntry.schema.sver; } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index bebdfb3b63..c1a1e7570e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -330,12 +330,12 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, ASSERT(pReadh->pDCols[0]->bitmapMode != 0); } - if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - if (pDataCol->bitmap) { + if (pDataCol->len > 0 && pDataCol->bitmap) { ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(pDataCol->pBitmap); tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap); tdDataColsSetBitmapI(pReadh->pDCols[0]); } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 341ab94ca4..a67f413ba7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -15,7 +15,7 @@ #include "tsdb.h" -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); +// static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { SSubmitMsgIter msgIter = {0}; @@ -54,7 +54,38 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * return 0; } -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { +#if 0 +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow *row, TSKEY minKey, TSKEY maxKey, + TSKEY now) { + TSKEY rowKey = TD_ROW_KEY(row); + if (rowKey < minKey || rowKey > maxKey) { + tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + " maxKey %" PRId64 " row key %" PRId64, + REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, + rowKey); + terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + return -1; + } + + return 0; +} +#endif + +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *row, TSKEY minKey, TSKEY maxKey, + TSKEY now) { + TSKEY rowKey = TD_ROW_KEY(row); + if (rowKey < minKey || rowKey > maxKey) { + tsdbError("vgId:%d table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + " maxKey %" PRId64 " row key %" PRId64, + REPO_ID(pTsdb), uid, now, minKey, maxKey, rowKey); + terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + return -1; + } + + return 0; +} + +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; @@ -112,14 +143,14 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { return -1; } } - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) { +#endif + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { + if (tsdbCheckRowRange(pTsdb, msgIter.uid, row, minKey, maxKey, now) < 0) { return -1; } } -#endif + } if (terrno != TSDB_CODE_SUCCESS) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 5c8cd362fd..9b6307c5f7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -631,6 +631,11 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); #endif + if (tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq) < 0) { + pRsp->code = terrno; + goto _exit; + } + // handle the request if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { pRsp->code = TSDB_CODE_INVALID_MSG; @@ -681,6 +686,9 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); tDecoderClear(&decoder); + } else { + submitBlkRsp.tblFName = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN); + sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname); } if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &submitBlkRsp) < 0) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d3045f68a1..23957d1a6b 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2883,6 +2883,110 @@ _return: CTG_API_LEAVE(code); } + + +int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver) { + *sver = -1; + + if (NULL == pCtg->dbCache) { + ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + int32_t tbType = 0; + uint64_t suid = 0; + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + STableMeta* tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + if (tbMeta) { + tbType = tbMeta->tableType; + suid = tbMeta->suid; + if (tbType != TSDB_CHILD_TABLE) { + *sver = tbMeta->sversion; + } + } + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); + + if (NULL == tbMeta) { + ctgReleaseDBCache(pCtg, dbCache); + return TSDB_CODE_SUCCESS; + } + + if (tbType != TSDB_CHILD_TABLE) { + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; + } + + ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, suid); + + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); + + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &suid, sizeof(suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("stb not in stbCache, suid:%"PRIx64, suid); + return TSDB_CODE_SUCCESS; + } + + if ((*stbMeta)->suid != suid) { + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseDBCache(pCtg, dbCache); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, suid, (*stbMeta)->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *sver = (*stbMeta)->sversion; + + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + + ctgReleaseDBCache(pCtg, dbCache); + + ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, tbType, dbFName, pTableName->tname); + + return TSDB_CODE_SUCCESS; +} + + +int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + SName name; + int32_t sver = 0; + int32_t tbNum = taosArrayGetSize(pTables); + for (int32_t i = 0; i < tbNum; ++i) { + STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); + tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + if (CTG_IS_SYS_DBNAME(name.dbname)) { + continue; + } + + ctgGetTbSverFromCache(pCtg, &name, &sver); + if (sver >= 0 && sver < pTb->sver) { + catalogRemoveTableMeta(pCtg, &name); //TODO REMOVE STB FROM CACHE + } + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + + int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { CTG_API_ENTER(); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 66073b70eb..35b81b135f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -106,7 +106,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { pMsg->contentLen = pMsg->contentLen; #endif - qDebugL("stream task string %s", (const char*)msg); + /*qDebugL("stream task string %s", (const char*)msg);*/ struct SSubplan* plan = NULL; int32_t code = qStringToSubplan(msg, &plan); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e42ced4669..3505af46e4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -869,8 +869,12 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SScalarParam dest = {.columnData = &idata}; - scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); - + int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pBlockList); + return code; + } + int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); @@ -5311,4 +5315,4 @@ int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t ke pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK);; return TSDB_CODE_SUCCESS; } - \ No newline at end of file + diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 802cdf7aad..4c23e4ba4b 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -118,21 +118,21 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_FLOAT) { - float va = strtod(a, NULL); + float va = taosStr2Float(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } - float vb = strtod(b, NULL); + float vb = taosStr2Float(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_DOUBLE) { - double va = strtod(a, NULL); + double va = taosStr2Double(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } - double vb = strtod(b, NULL); + double vb = taosStr2Double(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a438a1e843..894659ddd1 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -123,7 +123,7 @@ static bool checkAndSplitEndpoint(SAstCreateContext* pCxt, const SToken* pEp, ch pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENDPOINT); } else { strncpy(pFqdn, ep, pColon - ep); - *pPort = strtol(pColon + 1, NULL, 10); + *pPort = taosStr2Int32(pColon + 1, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } @@ -147,7 +147,7 @@ static bool checkPort(SAstCreateContext* pCxt, const SToken* pPortToken, int32_t if (NULL == pPortToken) { pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else { - *pPort = strtol(pPortToken->z, NULL, 10); + *pPort = taosStr2Int32(pPortToken->z, NULL, 10); if (*pPort >= UINT16_MAX || *pPort <= 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PORT); } @@ -476,9 +476,9 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); CHECK_OUT_OF_MEM(limitNode); - limitNode->limit = strtol(pLimit->z, NULL, 10); + limitNode->limit = taosStr2Int64(pLimit->z, NULL, 10); if (NULL != pOffset) { - limitNode->offset = strtol(pOffset->z, NULL, 10); + limitNode->offset = taosStr2Int64(pOffset->z, NULL, 10); } return (SNode*)limitNode; } @@ -694,59 +694,59 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { switch (type) { case DB_OPTION_BUFFER: - ((SDatabaseOptions*)pOptions)->buffer = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_CACHELAST: - ((SDatabaseOptions*)pOptions)->cachelast = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->cachelast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_COMP: - ((SDatabaseOptions*)pOptions)->compressionLevel = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_DAYS: { SToken* pToken = pVal; if (TK_NK_INTEGER == pToken->type) { - ((SDatabaseOptions*)pOptions)->daysPerFile = strtol(pToken->z, NULL, 10) * 1440; + ((SDatabaseOptions*)pOptions)->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440; } else { ((SDatabaseOptions*)pOptions)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken); } break; } case DB_OPTION_FSYNC: - ((SDatabaseOptions*)pOptions)->fsyncPeriod = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MAXROWS: - ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MINROWS: - ((SDatabaseOptions*)pOptions)->minRowsPerBlock = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_KEEP: ((SDatabaseOptions*)pOptions)->pKeep = pVal; break; case DB_OPTION_PAGES: - ((SDatabaseOptions*)pOptions)->pages = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PAGESIZE: - ((SDatabaseOptions*)pOptions)->pagesize = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PRECISION: copyStringFormStringToken((SToken*)pVal, ((SDatabaseOptions*)pOptions)->precisionStr, sizeof(((SDatabaseOptions*)pOptions)->precisionStr)); break; case DB_OPTION_REPLICA: - ((SDatabaseOptions*)pOptions)->replica = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_STRICT: - ((SDatabaseOptions*)pOptions)->strict = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->strict = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_WAL: - ((SDatabaseOptions*)pOptions)->walLevel = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_VGROUPS: - ((SDatabaseOptions*)pOptions)->numOfVgroups = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_SINGLE_STABLE: - ((SDatabaseOptions*)pOptions)->singleStable = strtol(((SToken*)pVal)->z, NULL, 10); + ((SDatabaseOptions*)pOptions)->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_RETENTIONS: ((SDatabaseOptions*)pOptions)->pRetentions = pVal; @@ -827,16 +827,16 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType sizeof(((STableOptions*)pOptions)->comment)); break; case TABLE_OPTION_DELAY: - ((STableOptions*)pOptions)->delay = strtol(((SToken*)pVal)->z, NULL, 10); + ((STableOptions*)pOptions)->delay = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case TABLE_OPTION_FILE_FACTOR: - ((STableOptions*)pOptions)->filesFactor = strtod(((SToken*)pVal)->z, NULL); + ((STableOptions*)pOptions)->filesFactor = taosStr2Float(((SToken*)pVal)->z, NULL); break; case TABLE_OPTION_ROLLUP: ((STableOptions*)pOptions)->pRollupFuncs = pVal; break; case TABLE_OPTION_TTL: - ((STableOptions*)pOptions)->ttl = strtol(((SToken*)pVal)->z, NULL, 10); + ((STableOptions*)pOptions)->ttl = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case TABLE_OPTION_SMA: ((STableOptions*)pOptions)->pSma = pVal; @@ -868,7 +868,7 @@ SDataType createDataType(uint8_t type) { } SDataType createVarLenDataType(uint8_t type, const SToken* pLen) { - SDataType dt = {.type = type, .precision = 0, .scale = 0, .bytes = strtol(pLen->z, NULL, 10)}; + SDataType dt = {.type = type, .precision = 0, .scale = 0, .bytes = taosStr2Int16(pLen->z, NULL, 10)}; return dt; } @@ -1119,7 +1119,7 @@ SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode) { SDropDnodeStmt* pStmt = (SDropDnodeStmt*)nodesMakeNode(QUERY_NODE_DROP_DNODE_STMT); CHECK_OUT_OF_MEM(pStmt); if (TK_NK_INTEGER == pDnode->type) { - pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnode->z, NULL, 10); } else { if (!checkAndSplitEndpoint(pCxt, pDnode, pStmt->fqdn, &pStmt->port)) { nodesDestroyNode(pStmt); @@ -1133,7 +1133,7 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const const SToken* pValue) { SAlterDnodeStmt* pStmt = nodesMakeNode(QUERY_NODE_ALTER_DNODE_STMT); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnode->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnode->z, NULL, 10); trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); if (NULL != pValue) { trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); @@ -1183,7 +1183,7 @@ SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { SCreateComponentNodeStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); ; return (SNode*)pStmt; } @@ -1191,7 +1191,7 @@ SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, co SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId) { SDropComponentNodeStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->dnodeId = strtol(pDnodeId->z, NULL, 10); + pStmt->dnodeId = taosStr2Int32(pDnodeId->z, NULL, 10); ; return (SNode*)pStmt; } @@ -1251,7 +1251,7 @@ SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* } SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal) { - ((SExplainOptions*)pOptions)->ratio = strtod(pVal->z, NULL); + ((SExplainOptions*)pOptions)->ratio = taosStr2Double(pVal->z, NULL); return pOptions; } @@ -1347,7 +1347,7 @@ SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId) { SKillStmt* pStmt = nodesMakeNode(type); CHECK_OUT_OF_MEM(pStmt); - pStmt->targetId = strtol(pId->z, NULL, 10); + pStmt->targetId = taosStr2Int32(pId->z, NULL, 10); return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 29951ccdee..34ae13580b 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -442,7 +442,7 @@ static bool isNullStr(SToken* pToken) { static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) { errno = 0; - *value = strtold(pToken->z, endPtr); + *value = taosStr2Double(pToken->z, endPtr); // not a valid integer number, return error if ((*endPtr - pToken->z) != pToken->n) { @@ -482,9 +482,9 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(pMsgBuf, ((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else if (pToken->type == TK_NK_FLOAT) { - return func(pMsgBuf, ((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); } else { return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 18e0843e92..0d214fd85e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -498,7 +498,7 @@ static int32_t parseTimeFromValueNode(SValueNode* pVal) { return TSDB_CODE_SUCCESS; } char* pEnd = NULL; - pVal->datum.i = strtoll(pVal->literal, &pEnd, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &pEnd, 10); return (NULL != pEnd && '\0' == *pEnd) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; } else { return TSDB_CODE_FAILED; @@ -527,61 +527,61 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD break; case TSDB_DATA_TYPE_TINYINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int8_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_SMALLINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int16_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_INT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int32_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; - pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); *(int64_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_UTINYINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint8_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_USMALLINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint16_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint32_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UBIGINT: { char* endPtr = NULL; - pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); *(uint64_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_FLOAT: { char* endPtr = NULL; - pVal->datum.d = strtold(pVal->literal, &endPtr); + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); *(float*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_DOUBLE: { char* endPtr = NULL; - pVal->datum.d = strtold(pVal->literal, &endPtr); + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); *(double*)&pVal->typeData = pVal->datum.d; break; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 795b5ca641..2d844e1842 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -3863,7 +3863,7 @@ static YYACTIONTYPE yy_reduce( { yymsp[1].minor.yy140 = 0; } break; case 245: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy140 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy140 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 246: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ { pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy617, &yymsp[-4].minor.yy105, yymsp[-2].minor.yy172, yymsp[-3].minor.yy172, yymsp[0].minor.yy172); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index f48095ecb8..db63c71d11 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -9,11 +9,11 @@ #include "tmsg.h" #include "tname.h" -SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; +SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; SQWorkerMgmt gQwMgmt = { - .lock = 0, - .qwRef = -1, - .qwNum = 0, + .lock = 0, + .qwRef = -1, + .qwNum = 0, }; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { @@ -110,9 +110,9 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) { QW_LOCK(QW_READ, &mgmt->schLock); - QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash)); + /*QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash));*/ - void * key = NULL; + void *key = NULL; size_t keyLen = 0; int32_t i = 0; SQWSchStatus *sch = NULL; @@ -127,7 +127,7 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt) { QW_UNLOCK(QW_READ, &mgmt->schLock); - QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash)); + /*QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash));*/ } char *qwPhaseStr(int32_t phase) { @@ -462,7 +462,7 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { } int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -499,7 +499,7 @@ _return: } int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -550,11 +550,11 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { int32_t code = 0; bool qcontinue = true; - SSDataBlock * pRes = NULL; + SSDataBlock *pRes = NULL; uint64_t useconds = 0; int32_t i = 0; int32_t execNum = 0; - qTaskInfo_t * taskHandle = &ctx->taskHandle; + qTaskInfo_t *taskHandle = &ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; while (true) { @@ -632,7 +632,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) return TSDB_CODE_QRY_OUT_OF_MEMORY; } - void * key = NULL; + void *key = NULL; size_t keyLen = 0; int32_t i = 0; STaskStatus status = {0}; @@ -719,8 +719,8 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void } int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { - int32_t code = 0; - SQWTaskCtx * ctx = NULL; + int32_t code = 0; + SQWTaskCtx *ctx = NULL; SRpcHandleInfo *dropConnection = NULL; SRpcHandleInfo *cancelConnection = NULL; @@ -925,13 +925,13 @@ _return: } int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { - int32_t code = 0; - bool queryRsped = false; - SSubplan* plan = NULL; - SQWPhaseInput input = {0}; - qTaskInfo_t pTaskInfo = NULL; - DataSinkHandle sinkHandle = NULL; - SQWTaskCtx * ctx = NULL; + int32_t code = 0; + bool queryRsped = false; + SSubplan *plan = NULL; + SQWPhaseInput input = {0}; + qTaskInfo_t pTaskInfo = NULL; + DataSinkHandle sinkHandle = NULL; + SQWTaskCtx *ctx = NULL; QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); @@ -944,7 +944,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex ctx->ctrlConnInfo = qwMsg->connInfo; - QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); + /*QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg);*/ code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { @@ -1055,10 +1055,10 @@ _return: } int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { - SQWTaskCtx * ctx = NULL; + SQWTaskCtx *ctx = NULL; int32_t code = 0; SQWPhaseInput input = {0}; - void * rsp = NULL; + void *rsp = NULL; int32_t dataLen = 0; bool queryEnd = false; @@ -1138,8 +1138,8 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; int32_t dataLen = 0; bool locked = false; - SQWTaskCtx * ctx = NULL; - void * rsp = NULL; + SQWTaskCtx *ctx = NULL; + void *rsp = NULL; SQWPhaseInput input = {0}; QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL)); @@ -1274,7 +1274,7 @@ _return: int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); @@ -1300,7 +1300,7 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; if (qwMsg->code) { QW_RET(qwProcessHbLinkBroken(mgmt, qwMsg, req)); @@ -1338,28 +1338,28 @@ _return: qwMsg->connInfo.handle = NULL; } - QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); + /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));*/ QW_RET(TSDB_CODE_SUCCESS); } void qwProcessHbTimerEvent(void *param, void *tmrId) { - SQWHbParam* hbParam = (SQWHbParam*)param; + SQWHbParam *hbParam = (SQWHbParam *)param; if (hbParam->qwrId != atomic_load_32(&gQwMgmt.qwRef)) { return; } - int64_t refId = hbParam->refId; + int64_t refId = hbParam->refId; SQWorker *mgmt = qwAcquire(refId); if (NULL == mgmt) { QW_DLOG("qwAcquire %" PRIx64 "failed", refId); taosMemoryFree(param); return; } - + SQWSchStatus *sch = NULL; int32_t taskNum = 0; - SQWHbInfo * rspList = NULL; + SQWHbInfo *rspList = NULL; int32_t code = 0; qwDbgDumpMgmtInfo(mgmt); @@ -1383,7 +1383,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { return; } - void * key = NULL; + void *key = NULL; size_t keyLen = 0; int32_t i = 0; @@ -1413,29 +1413,27 @@ _return: for (int32_t j = 0; j < i; ++j) { qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); - QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code), - (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0)); + /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),*/ + /*(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));*/ tFreeSSchedulerHbRsp(&rspList[j].rsp); } taosMemoryFreeClear(rspList); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); - qwRelease(refId); + qwRelease(refId); } void qwCloseRef(void) { taosWLockLatch(&gQwMgmt.lock); if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) { taosCloseRef(gQwMgmt.qwRef); - gQwMgmt.qwRef= -1; + gQwMgmt.qwRef = -1; } taosWUnLockLatch(&gQwMgmt.lock); } -void qwDestroySchStatus(SQWSchStatus *pStatus) { - taosHashCleanup(pStatus->tasksHash); -} +void qwDestroySchStatus(SQWSchStatus *pStatus) { taosHashCleanup(pStatus->tasksHash); } void qwDestroyImpl(void *pMgmt) { SQWorker *mgmt = (SQWorker *)pMgmt; @@ -1454,12 +1452,12 @@ void qwDestroyImpl(void *pMgmt) { SQWSchStatus *sch = (SQWSchStatus *)pIter; qwDestroySchStatus(sch); pIter = taosHashIterate(mgmt->schHash, pIter); - } + } taosHashCleanup(mgmt->schHash); taosMemoryFree(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); qwCloseRef(); } @@ -1467,7 +1465,7 @@ void qwDestroyImpl(void *pMgmt) { int32_t qwOpenRef(void) { taosWLockLatch(&gQwMgmt.lock); if (gQwMgmt.qwRef < 0) { - gQwMgmt.qwRef= taosOpenRef(100, qwDestroyImpl); + gQwMgmt.qwRef = taosOpenRef(100, qwDestroyImpl); if (gQwMgmt.qwRef < 0) { taosWUnLockLatch(&gQwMgmt.lock); qError("init qworker ref failed"); @@ -1475,14 +1473,14 @@ int32_t qwOpenRef(void) { } } taosWUnLockLatch(&gQwMgmt.lock); - + return TSDB_CODE_SUCCESS; } void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { int32_t paramIdx = 0; int32_t newParamIdx = 0; - + while (true) { paramIdx = atomic_load_32(&gQwMgmt.paramIdx); if (paramIdx == tListLen(gQwMgmt.param)) { @@ -1490,7 +1488,7 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { } else { newParamIdx = paramIdx + 1; } - + if (paramIdx == atomic_val_compare_exchange_32(&gQwMgmt.paramIdx, paramIdx, newParamIdx)) { break; } @@ -1577,12 +1575,12 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW SQWHbParam *param = NULL; qwSetHbParam(mgmt->refId, ¶m); - mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void*)param, mgmt->timer); + mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void *)param, mgmt->timer); if (NULL == mgmt->hbTimer) { qError("start hb timer failed"); QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + *qWorkerMgmt = mgmt; qDebug("qworker initialized for node, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); @@ -1599,9 +1597,9 @@ _return: taosTmrCleanUp(mgmt->timer); taosMemoryFreeClear(mgmt); - atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); + atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); } - + QW_RET(code); } @@ -1678,7 +1676,7 @@ int32_t qwUpdateSchLastAccess(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64 } int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -1705,7 +1703,7 @@ int32_t qwGetTaskStatus(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId } int32_t qwCancelTask(SQWorker *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { - SQWSchStatus * sch = NULL; + SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d4a88622e2..45742189d5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -724,7 +724,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp case TSDB_DATA_TYPE_BIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { memcpy(output, varDataVal(input), varDataLen(input)); - *(int64_t *)output = strtoll(output, NULL, 10); + *(int64_t *)output = taosStr2Int64(output, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); @@ -733,7 +733,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp return TSDB_CODE_FAILED; } newBuf[len] = 0; - *(int64_t *)output = strtoll(newBuf, NULL, 10); + *(int64_t *)output = taosStr2Int64(newBuf, NULL, 10); taosMemoryFree(newBuf); } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); @@ -743,7 +743,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp case TSDB_DATA_TYPE_UBIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { memcpy(output, varDataVal(input), varDataLen(input)); - *(uint64_t *)output = strtoull(output, NULL, 10); + *(uint64_t *)output = taosStr2UInt64(output, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); @@ -752,7 +752,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp return TSDB_CODE_FAILED; } newBuf[len] = 0; - *(uint64_t *)output = strtoull(newBuf, NULL, 10); + *(uint64_t *)output = taosStr2UInt64(newBuf, NULL, 10); taosMemoryFree(newBuf); } else { GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c9fcaeb32e..19453bf760 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -92,7 +92,7 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int tmp[len] = 0; ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); - double value = strtod(tmp, NULL); + double value = taosStr2Double(tmp, NULL); *((double *)outData) = value; taosMemoryFreeClear(tmp); @@ -267,22 +267,22 @@ static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam* pOut, int32_t r static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_TINYINT: { - int8_t value = (int8_t)strtoll(buf, NULL, 10); + int8_t value = (int8_t)taosStr2Int8(buf, NULL, 10); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t value = (int16_t)strtoll(buf, NULL, 10); + int16_t value = (int16_t)taosStr2Int16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; } case TSDB_DATA_TYPE_INT: { - int32_t value = (int32_t)strtoll(buf, NULL, 10); + int32_t value = (int32_t)taosStr2Int32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t value = (int64_t)strtoll(buf, NULL, 10); + int64_t value = (int64_t)taosStr2Int64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; } @@ -292,22 +292,22 @@ static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t rowI static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t rowIndex) { switch (pOut->columnData->info.type) { case TSDB_DATA_TYPE_UTINYINT: { - uint8_t value = (uint8_t)strtoull(buf, NULL, 10); + uint8_t value = (uint8_t)taosStr2UInt8(buf, NULL, 10); colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*)&value); break; } case TSDB_DATA_TYPE_USMALLINT: { - uint16_t value = (uint16_t)strtoull(buf, NULL, 10); + uint16_t value = (uint16_t)taosStr2UInt16(buf, NULL, 10); colDataAppendInt16(pOut->columnData, rowIndex, (int16_t*)&value); break; } case TSDB_DATA_TYPE_UINT: { - uint32_t value = (uint32_t)strtoull(buf, NULL, 10); + uint32_t value = (uint32_t)taosStr2UInt32(buf, NULL, 10); colDataAppendInt32(pOut->columnData, rowIndex, (int32_t*)&value); break; } case TSDB_DATA_TYPE_UBIGINT: { - uint64_t value = (uint64_t)strtoull(buf, NULL, 10); + uint64_t value = (uint64_t)taosStr2UInt64(buf, NULL, 10); colDataAppendInt64(pOut->columnData, rowIndex, (int64_t*)&value); break; } @@ -315,12 +315,12 @@ static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t ro } static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t rowIndex) { - double value = strtod(buf, NULL); + double value = taosStr2Double(buf, NULL); colDataAppendDouble(pOut->columnData, rowIndex, &value); } static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowIndex) { - int64_t value = strtoll(buf, NULL, 10); + int64_t value = taosStr2Int64(buf, NULL, 10); bool v = (value != 0)? true:false; colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index a90fb7fc2e..5a6fcee759 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -39,6 +39,12 @@ enum { SCH_WRITE, }; +typedef enum { + SCH_RES_TYPE_QUERY, + SCH_RES_TYPE_FETCH, +} SCH_RES_TYPE; + + typedef struct SSchTrans { void *transInst; void *transHandle; @@ -159,7 +165,6 @@ typedef struct SSchTask { typedef struct SSchJobAttr { EExplainMode explainMode; - bool needRes; bool syncSchedule; bool queryJob; bool needFlowCtrl; @@ -192,6 +197,7 @@ typedef struct SSchJob { int32_t errCode; SArray *errList; // SArray SRWLatch resLock; + SCH_RES_TYPE resType; void *resData; //TODO free it or not int32_t resNumOfRows; const char *sql; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index bdf674ab93..9354c1a875 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -70,7 +70,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * } int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray *pNodeList, const char *sql, - int64_t startTs, bool needRes, bool syncSchedule) { + int64_t startTs, bool syncSchedule) { int32_t code = 0; int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); @@ -81,7 +81,6 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *transport, SArray pJob->attr.explainMode = pDag->explainInfo.mode; pJob->attr.syncSchedule = syncSchedule; - pJob->attr.needRes = needRes; pJob->transport = transport; pJob->sql = sql; @@ -1059,6 +1058,8 @@ _return: int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); + pJob->resType = SCH_RES_TYPE_FETCH; + atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); atomic_store_ptr(&pJob->resData, pRsp); @@ -1179,23 +1180,20 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows); - if (pJob->attr.needRes) { - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->resData) { - SSubmitRsp *sum = pJob->resData; - sum->affectedRows += rsp->affectedRows; - sum->nBlocks += rsp->nBlocks; - sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); - memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); - taosMemoryFree(rsp->pBlocks); - taosMemoryFree(rsp); - } else { - pJob->resData = rsp; - } - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + pJob->resType = SCH_RES_TYPE_QUERY; + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->resData) { + SSubmitRsp *sum = pJob->resData; + sum->affectedRows += rsp->affectedRows; + sum->nBlocks += rsp->nBlocks; + sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); + memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + taosMemoryFree(rsp->pBlocks); + taosMemoryFree(rsp); } else { - tFreeSSubmitRsp(rsp); + pJob->resData = rsp; } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); @@ -2412,7 +2410,7 @@ void schFreeJobImpl(void *job) { } static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, - int64_t startTs, bool needRes, bool syncSchedule) { + int64_t startTs, bool syncSchedule) { qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); if (pNodeList == NULL || taosArrayGetSize(pNodeList) <= 0) { @@ -2421,7 +2419,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD int32_t code = 0; SSchJob *pJob = NULL; - SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, needRes, syncSchedule)); + SCH_ERR_JRET(schInitJob(&pJob, pDag, transport, pNodeList, sql, startTs, syncSchedule)); SCH_ERR_JRET(schLaunchJob(pJob)); @@ -2463,6 +2461,8 @@ int32_t schExecStaticExplain(void *transport, SArray *pNodeList, SQueryPlan *pDa SCH_ERR_JRET(qExecStaticExplain(pDag, (SRetrieveTableRsp **)&pJob->resData)); + pJob->resType = SCH_RES_TYPE_FETCH; + int64_t refId = taosAddRef(schMgmt.jobRef, pJob); if (refId < 0) { SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); @@ -2535,7 +2535,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, - int64_t startTs, bool needRes, SQueryResult *pRes) { + int64_t startTs, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -2543,14 +2543,14 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, nodeList, pDag, pJob, sql, true)); } else { - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, needRes, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, startTs, true)); } SSchJob *job = schAcquireJob(*pJob); pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; - if (needRes) { + if (SCH_RES_TYPE_QUERY == job->resType) { pRes->res = job->resData; job->resData = NULL; } @@ -2568,7 +2568,7 @@ int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan *pD if (EXPLAIN_MODE_STATIC == pDag->explainInfo.mode) { SCH_ERR_RET(schExecStaticExplain(transport, pNodeList, pDag, pJob, sql, false)); } else { - SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, 0, false)); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 09ecd9fffd..fc0e05aaf1 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -985,7 +985,7 @@ TEST(insertTest, normalCase) { taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, false, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &insertJobRefId, "insert into tb values(now,1)", 0, &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 521b61e3c8..5912fd800c 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,7 +22,7 @@ extern "C" { #endif -typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int32_t kLen1, const void *pKey2, int32_t kLen2); // exposed types typedef struct STDB TDB; @@ -31,42 +31,42 @@ typedef struct STBC TBC; typedef struct STxn TXN; // TDB -int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb); -int tdbClose(TDB *pDb); -int tdbBegin(TDB *pDb, TXN *pTxn); -int tdbCommit(TDB *pDb, TXN *pTxn); +int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb); +int32_t tdbClose(TDB *pDb); +int32_t tdbBegin(TDB *pDb, TXN *pTxn); +int32_t tdbCommit(TDB *pDb, TXN *pTxn); // TTB -int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); -int tdbTbClose(TTB *pTb); -int tdbTbDrop(TTB *pTb); -int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); -int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); -int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); -int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); +int32_t tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); +int32_t tdbTbClose(TTB *pTb); +int32_t tdbTbDrop(TTB *pTb); +int32_t tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int32_t tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); +int32_t tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int32_t tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); +int32_t tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); // TBC -int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); -int tdbTbcClose(TBC *pTbc); -int tdbTbcIsValid(TBC *pTbc); -int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c); -int tdbTbcMoveToFirst(TBC *pTbc); -int tdbTbcMoveToLast(TBC *pTbc); -int tdbTbcMoveToNext(TBC *pTbc); -int tdbTbcMoveToPrev(TBC *pTbc); -int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); -int tdbTbcDelete(TBC *pTbc); -int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert); +int32_t tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); +int32_t tdbTbcClose(TBC *pTbc); +int32_t tdbTbcIsValid(TBC *pTbc); +int32_t tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c); +int32_t tdbTbcMoveToFirst(TBC *pTbc); +int32_t tdbTbcMoveToLast(TBC *pTbc); +int32_t tdbTbcMoveToNext(TBC *pTbc); +int32_t tdbTbcMoveToPrev(TBC *pTbc); +int32_t tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); +int32_t tdbTbcDelete(TBC *pTbc); +int32_t tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int32_t tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert); // TXN #define TDB_TXN_WRITE 0x1 #define TDB_TXN_READ_UNCOMMITTED 0x2 -int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, - int flags); -int tdbTxnClose(TXN *pTxn); +int32_t tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), + void *xArg, int flags); +int32_t tdbTxnClose(TXN *pTxn); // other void tdbFree(void *); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 912c1ecaaf..c06f7305ab 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,7 +15,7 @@ #include "tdbInt.h" -int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { +int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb) { TDB *pDb; int dsize; int zsize; @@ -25,7 +25,7 @@ int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { *ppDb = NULL; - dsize = strlen(rootDir); + dsize = strlen(dbname); zsize = sizeof(*pDb) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; pPtr = (uint8_t *)tdbOsCalloc(1, zsize); @@ -36,16 +36,16 @@ int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { pDb = (TDB *)pPtr; pPtr += sizeof(*pDb); // pDb->rootDir - pDb->rootDir = pPtr; - memcpy(pDb->rootDir, rootDir, dsize); - pDb->rootDir[dsize] = '\0'; + pDb->dbName = pPtr; + memcpy(pDb->dbName, dbname, dsize); + pDb->dbName[dsize] = '\0'; pPtr = pPtr + dsize + 1; // pDb->jfname - pDb->jfname = pPtr; - memcpy(pDb->jfname, rootDir, dsize); - pDb->jfname[dsize] = '/'; - memcpy(pDb->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); - pDb->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; + pDb->jnName = pPtr; + memcpy(pDb->jnName, dbname, dsize); + pDb->jnName[dsize] = '/'; + memcpy(pDb->jnName + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); + pDb->jnName[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; pDb->jfd = -1; @@ -62,7 +62,7 @@ int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { } memset(pDb->pgrHash, 0, tsize); - mkdir(rootDir, 0755); + mkdir(dbname, 0755); *ppDb = pDb; return 0; diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index ce7da0b471..7211fe4926 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -24,7 +24,7 @@ struct STBC { SBTC btc; }; -int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { +int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { TTB *pTb; SPager *pPager; int ret; @@ -42,9 +42,9 @@ int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn // pTb->pEnv pTb->pEnv = pEnv; - pPager = tdbEnvGetPager(pEnv, fname); + pPager = tdbEnvGetPager(pEnv, tbname); if (pPager == NULL) { - snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname); ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); if (ret < 0) { return -1; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index bb59a7fc61..2c79f39bc0 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -335,8 +335,8 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { } struct STDB { - char *rootDir; - char *jfname; + char *dbName; + char *jnName; int jfd; SPCache *pCache; SPager *pgrList; diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index ad6cfc8b95..90b8e9dd8a 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -27,6 +27,9 @@ if(BUILD_ADDR2LINE) os PUBLIC addr2line dl z ) endif () +if(CHECK_STR2INT_ERROR) + add_definitions(-DTD_CHECK_STR_TO_INT_ERROR) +endif() target_link_libraries( os PUBLIC pthread ) diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 7dbd301913..da1fbd364f 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -254,4 +254,103 @@ char *taosStrCaseStr(const char *str, const char *pattern) { } } return NULL; +} + +int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix) { + int64_t tmp = strtoll(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); +#endif + return tmp; +} + +uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix) { + uint64_t tmp = strtoull(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); +#endif + return tmp; +} + +int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); +#endif + return tmp; +} + +uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); +#endif + return tmp; +} + +int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp >= SHRT_MIN); + assert(tmp <= SHRT_MAX); +#endif + return (int16_t)tmp; +} + +uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtoul(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp <= USHRT_MAX); +#endif + return (uint16_t)tmp; +} + +int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix) { + int32_t tmp = strtol(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp >= SCHAR_MIN); + assert(tmp <= SCHAR_MAX); +#endif + return tmp; +} + +uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix) { + uint32_t tmp = strtoul(str, pEnd, radix); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp <= UCHAR_MAX); +#endif + return tmp; +} + +double taosStr2Double(const char *str, char** pEnd) { + double tmp = strtod(str, pEnd); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp != HUGE_VAL); +#endif + return tmp; +} + +float taosStr2Float(const char *str, char** pEnd) { + float tmp = strtof(str, pEnd); +#ifdef TD_CHECK_STR_TO_INT_ERROR + assert(errno != ERANGE); + assert(errno != EINVAL); + assert(tmp != HUGE_VALF); + assert(tmp != NAN); +#endif + return tmp; } \ No newline at end of file diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index c5cd968fd9..b15c188f04 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -187,7 +187,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal sscanf(p,"%lld",pVal); #else // sscanf(p,"%ld",pVal); - *pVal = strtol(p, NULL, 10); + *pVal = taosStr2Int64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; } @@ -222,7 +222,7 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV sscanf(p,"%llu",pVal); #else // sscanf(p,"%ld",pVal); - *pVal = strtoul(p, NULL, 10); + *pVal = taosStr2UInt64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; } diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 13637b8fe4..4ce668e6ba 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -185,10 +185,10 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it pKey = SL_GET_NODE_KEY(pSkipList, p); compare = pSkipList->comparFn(pKey, pDataKey); - if (compare >= 0) { - if (compare == 0 && !hasDup) hasDup = true; + if (compare > 0) { break; } else { + if (compare == 0 && !hasDup) hasDup = true; px = p; p = SL_NODE_GET_FORWARD_POINTER(px, i); } diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index adbab04e07..bdda7c453b 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -57,7 +57,7 @@ class TDSql: tdLog.notice("'reset query cache' is not supported") s = 'drop database if exists db' self.cursor.execute(s) - s = 'create database db' + s = 'create database db days 300' self.cursor.execute(s) s = 'use db' self.cursor.execute(s) diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 68860dc2cb..dcd88e5a0c 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -5,7 +5,7 @@ sleep 500 sql connect print =============== create database -sql create database d0 +sql create database d0 days 300 sql use d0 print =============== create super table and child table @@ -254,4 +254,4 @@ endi #sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/tstream/basic1.sim index cb084ad537..0997e313c8 100644 --- a/tests/script/tsim/tstream/basic1.sim +++ b/tests/script/tsim/tstream/basic1.sim @@ -24,7 +24,7 @@ sql insert into t1 values(1648791233002,3,2,3,2.1); sql insert into t1 values(1648791243003,4,2,3,3.1); sql insert into t1 values(1648791213004,4,2,3,4.1); sleep 1000 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; if $rows != 4 then print ======$rows @@ -137,7 +137,7 @@ endi sql insert into t1 values(1648791223001,12,14,13,11.1); sleep 500 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; if $rows != 4 then print ======$rows @@ -250,7 +250,7 @@ endi sql insert into t1 values(1648791223002,12,14,13,11.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 2 then @@ -280,7 +280,7 @@ endi sql insert into t1 values(1648791223003,12,14,13,11.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 3 then @@ -312,7 +312,7 @@ sql insert into t1 values(1648791223001,1,1,1,1.1); sql insert into t1 values(1648791223002,2,2,2,2.1); sql insert into t1 values(1648791223003,3,3,3,3.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 3 then @@ -344,7 +344,7 @@ sql insert into t1 values(1648791233003,3,2,3,2.1); sql insert into t1 values(1648791233002,5,6,7,8.1); sql insert into t1 values(1648791233002,3,2,3,2.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 2 if $data21 != 2 then @@ -374,7 +374,7 @@ endi sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 0 if $data01 != 4 then @@ -404,58 +404,58 @@ endi sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); sleep 100 -sql select _wstartts, c1, c2 ,c3 ,c4, c5 from streamt; +sql select `_wstartts`, c1, c2 ,c3 ,c4, c5 from streamt; # row 1 if $data11 != 4 then print ======$data11 - # return -1 + return -1 endi if $data12 != 4 then print ======$data12 - # return -1 + return -1 endi if $data13 != 10 then print ======$data13 - # return -1 + return -1 endi if $data14 != 3 then print ======$data14 - # return -1 + return -1 endi if $data15 != 1 then print ======$data15 - # return -1 + return -1 endi # row 2 if $data21 != 4 then print ======$data21 - # return -1 + return -1 endi if $data22 != 4 then print ======$data22 - # return -1 + return -1 endi if $data23 != 15 then print ======$data23 - # return -1 + return -1 endi if $data24 != 4 then print ======$data24 - # return -1 + return -1 endi if $data25 != 3 then print ======$data25 - # return -1 + return -1 endi diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index dc1a77c231..96d7741897 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -368,7 +368,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { /*msg_process(tmqmessage);*/ taos_free_result(tmqmessage); - if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); + if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit_sync(tmq, NULL); } } diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index cf113369bc..8322c1a60c 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -37,10 +37,10 @@ typedef struct { TdThread thread; int32_t consumerId; - int32_t ifManualCommit; - //int32_t autoCommitIntervalMs; // 1000 ms - //char autoCommit[8]; // true, false - //char autoOffsetRest[16]; // none, earliest, latest + int32_t ifManualCommit; + // int32_t autoCommitIntervalMs; // 1000 ms + // char autoCommit[8]; // true, false + // char autoOffsetRest[16]; // none, earliest, latest int32_t ifCheckData; int64_t expectMsgCnt; @@ -99,21 +99,15 @@ static void printHelp() { } void initLogFile() { - time_t now; - struct tm curTime; - char filename[256]; + time_t now; + struct tm curTime; + char filename[256]; - now = taosTime(NULL); + now = taosTime(NULL); taosLocalTime(&now, &curTime); - sprintf(filename,"%s/../log/tmqlog_%04d-%02d-%02d %02d-%02d-%02d.txt", - configDir, - curTime.tm_year+1900, - curTime.tm_mon+1, - curTime.tm_mday, - curTime.tm_hour, - curTime.tm_min, - curTime.tm_sec); - //sprintf(filename, "%s/../log/tmqlog.txt", configDir); + sprintf(filename, "%s/../log/tmqlog_%04d-%02d-%02d %02d-%02d-%02d.txt", configDir, curTime.tm_year + 1900, + curTime.tm_mon + 1, curTime.tm_mday, curTime.tm_hour, curTime.tm_min, curTime.tm_sec); + // sprintf(filename, "%s/../log/tmqlog.txt", configDir); TdFilePtr pFile = taosOpenFile(filename, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", filename); @@ -137,9 +131,9 @@ void saveConfigToLogFile() { for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - //taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); - //taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); - //taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); + // taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); + // taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); + // taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); taosFprintfFile(g_fp, " Topics: "); for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); @@ -234,17 +228,17 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) while (1) { TAOS_ROW row = taos_fetch_row(msg); - if (row == NULL) break; + if (row == NULL) break; - TAOS_FIELD* fields = taos_fetch_fields(msg); + TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); - + taos_print_row(buf, row, fields, numOfFields); - - if (0 != g_stConfInfo.showRowFlag) { + + if (0 != g_stConfInfo.showRowFlag) { taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); } - + totalRows++; } @@ -276,7 +270,7 @@ void build_consumer(SThreadInfo* pInfo) { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); - //tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -299,7 +293,7 @@ void build_consumer(SThreadInfo* pInfo) { pInfo->tmq = tmq_consumer_new(conf, NULL, 0); tmq_conf_destroy(conf); - + return; } @@ -322,10 +316,10 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); - time_t tTime = taosGetTimestampSec(); + time_t tTime = taosGetTimestampSec(); struct tm tm = *taosLocalTime(&tTime, NULL); taosFprintfFile(g_fp, "# save result: %d-%02d-%02d %02d:%02d:%02d, sql: %s\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, sqlStr); + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, sqlStr); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { @@ -357,11 +351,11 @@ void loop_consume(SThreadInfo* pInfo) { totalMsgs++; if (totalRows >= pInfo->expectMsgCnt) { - taosFprintfFile(g_fp, "==== totalRows >= pInfo->expectMsgCnt, so break\n"); + taosFprintfFile(g_fp, "==== totalRows >= pInfo->expectMsgCnt, so break\n"); break; } - } else { - taosFprintfFile(g_fp, "==== delay over time, so break\n"); + } else { + taosFprintfFile(g_fp, "==== delay over time, so break\n"); break; } } @@ -389,7 +383,7 @@ void* consumeThreadFunc(void* param) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - + tmq_list_destroy(pInfo->topicList); pInfo->topicList = NULL; @@ -397,17 +391,18 @@ void* consumeThreadFunc(void* param) { if (pInfo->ifManualCommit) { taosFprintfFile(g_fp, "tmq_commit() manual commit when consume end.\n"); - pPrint("tmq_commit() manual commit when consume end.\n"); - tmq_commit(pInfo->tmq, NULL, 0); + pPrint("tmq_commit() manual commit when consume end.\n"); + /*tmq_commit(pInfo->tmq, NULL, 0);*/ + tmq_commit_sync(pInfo->tmq, NULL); } - + err = tmq_unsubscribe(pInfo->tmq); if (err) { pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); pInfo->consumeMsgCnt = -1; return NULL; } - + err = tmq_consumer_close(pInfo->tmq); if (err) { pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); @@ -485,9 +480,9 @@ int32_t getConsumeInfo() { int32_t* lengths = taos_fetch_lengths(pRes); // set default value - //g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; - //memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); - //memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); + // g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; + // memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); + // memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); for (int i = 0; i < num_fields; ++i) { if (row[i] == NULL || 0 == i) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9f9c8821b0..21474c316f 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -195,16 +195,16 @@ void shellRunSingleCommandImp(char *command) { et = taosGetTimestampUs(); if (error_no == 0) { - printf("Query OK, %d row(s) in set (%.6fs)\n", numOfRows, (et - st) / 1E6); + printf("Query OK, %d rows affected (%.6fs)\n", numOfRows, (et - st) / 1E6); } else { - printf("Query interrupted (%s), %d row(s) in set (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); + printf("Query interrupted (%s), %d rows affected (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); } taos_free_result(pSql); } else { int32_t num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); + printf("Query OK, %d of %d rows affected (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); } printf("\n");