xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into enh/TD-31535/assert

This commit is contained in:
xinsheng Ren 2024-08-22 17:17:04 +08:00 committed by GitHub
parent 5ebe3e2672 caf737ab19
commit e28d8946c6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
69 changed files with 1339 additions and 2648 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/examples/c/with_reqid_demo.c
View File

@ -33,7 +33,8 @@ static int DemoWithReqId() {
// connect // connect
TAOS *taos = taos_connect(host, user, password, NULL, port); TAOS *taos = taos_connect(host, user, password, NULL, port);
if (taos == NULL) { if (taos == NULL) {
fprintf(stderr, "Failed to connect to %s:%hu, ErrCode: 0x%x, ErrMessage: %s.\n", host, port, taos_errno(NULL), taos_errstr(NULL)); fprintf(stderr, "Failed to connect to %s:%hu, ErrCode: 0x%x, ErrMessage: %s.\n", host, port, taos_errno(NULL),
taos_errstr(NULL));
taos_cleanup(); taos_cleanup();
return -1; return -1;
} }
@ -44,7 +45,8 @@ static int DemoWithReqId() {
TAOS_RES *result = taos_query_with_reqid(taos, sql, reqid); TAOS_RES *result = taos_query_with_reqid(taos, sql, reqid);
code = taos_errno(result); code = taos_errno(result);
if (code != 0) { if (code != 0) {
fprintf(stderr, "Failed to execute sql with reqId: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code, taos_errstr(result)); fprintf(stderr, "Failed to execute sql with QID: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code,
taos_errstr(result));
taos_close(taos); taos_close(taos);
taos_cleanup(); taos_cleanup();
return -1; return -1;
@ -73,6 +75,4 @@ static int DemoWithReqId() {
// ANCHOR_END: with_reqid // ANCHOR_END: with_reqid
} }
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) { return DemoWithReqId(); }
return DemoWithReqId();
}

3
include/libs/monitor/monitor.h
View File

@ -30,6 +30,8 @@ extern "C" {
#define MON_VER_LEN 12 #define MON_VER_LEN 12
#define MON_LOG_LEN 1024 #define MON_LOG_LEN 1024
#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows"
typedef struct { typedef struct {
int64_t ts; int64_t ts;
ELogLevel level; ELogLevel level;
@ -216,6 +218,7 @@ typedef struct {
} SDmNotifyHandle; } SDmNotifyHandle;
int32_t monInit(const SMonCfg *pCfg); int32_t monInit(const SMonCfg *pCfg);
void monInitVnode();
void monCleanup(); void monCleanup();
void monRecordLog(int64_t ts, ELogLevel level, const char *content); void monRecordLog(int64_t ts, ELogLevel level, const char *content);
int32_t monGetLogs(SMonLogs *logs); int32_t monGetLogs(SMonLogs *logs);

129
include/util/texception.h
View File

@ -1,129 +0,0 @@
/*
* Copyright (c) 2020 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_EXCEPTION_H_
#define _TD_UTIL_EXCEPTION_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
/*
* cleanup actions
*/
typedef struct SCleanupAction {
bool failOnly;
uint8_t wrapper;
uint16_t reserved;
void* func;
union {
void* Ptr;
bool Bool;
char Char;
int8_t Int8;
uint8_t Uint8;
int16_t Int16;
uint16_t Uint16;
int32_t Int;
uint32_t Uint;
int32_t Int32;
uint32_t Uint32;
int64_t Int64;
uint64_t Uint64;
float Float;
double Double;
} arg1, arg2;
} SCleanupAction;
/*
* exception hander registration
*/
typedef struct SExceptionNode {
struct SExceptionNode* prev;
jmp_buf jb;
int32_t code;
int32_t maxCleanupAction;
int32_t numCleanupAction;
SCleanupAction* cleanupActions;
} SExceptionNode;
// functions & macros for auto-cleanup
void cleanupPush_void_ptr_ptr(bool failOnly, void* func, void* arg1, void* arg2);
void cleanupPush_void_ptr_bool(bool failOnly, void* func, void* arg1, bool arg2);
void cleanupPush_void_ptr(bool failOnly, void* func, void* arg);
void cleanupPush_int_int(bool failOnly, void* func, int32_t arg);
void cleanupPush_void(bool failOnly, void* func);
void cleanupPush_int_ptr(bool failOnly, void* func, void* arg);
int32_t cleanupGetActionCount();
void cleanupExecuteTo(int32_t anchor, bool failed);
void cleanupExecute(SExceptionNode* node, bool failed);
bool cleanupExceedLimit();
#define CLEANUP_PUSH_VOID_PTR_PTR(failOnly, func, arg1, arg2) \
cleanupPush_void_ptr_ptr((failOnly), (void*)(func), (void*)(arg1), (void*)(arg2))
#define CLEANUP_PUSH_VOID_PTR_BOOL(failOnly, func, arg1, arg2) \
cleanupPush_void_ptr_bool((failOnly), (void*)(func), (void*)(arg1), (bool)(arg2))
#define CLEANUP_PUSH_VOID_PTR(failOnly, func, arg) cleanupPush_void_ptr((failOnly), (void*)(func), (void*)(arg))
#define CLEANUP_PUSH_INT_INT(failOnly, func, arg) cleanupPush_void_ptr((failOnly), (void*)(func), (int32_t)(arg))
#define CLEANUP_PUSH_VOID(failOnly, func) cleanupPush_void((failOnly), (void*)(func))
#define CLEANUP_PUSH_INT_PTR(failOnly, func, arg) cleanupPush_int_ptr((failOnly), (void*)(func), (void*)(arg))
#define CLEANUP_PUSH_FREE(failOnly, arg) cleanupPush_void_ptr((failOnly), free, (void*)(arg))
#define CLEANUP_PUSH_CLOSE(failOnly, arg) cleanupPush_int_int((failOnly), close, (int32_t)(arg))
#define CLEANUP_PUSH_FCLOSE(failOnly, arg) cleanupPush_int_ptr((failOnly), fclose, (void*)(arg))
#define CLEANUP_GET_ANCHOR() cleanupGetActionCount()
#define CLEANUP_EXECUTE_TO(anchor, failed) cleanupExecuteTo((anchor), (failed))
#define CLEANUP_EXCEED_LIMIT() cleanupExceedLimit()
// functions & macros for exception handling
void exceptionPushNode(SExceptionNode* node);
int32_t exceptionPopNode();
void exceptionThrow(int32_t code);
#define TRY(maxCleanupActions) \
do { \
SExceptionNode exceptionNode = {0}; \
SCleanupAction cleanupActions[(maxCleanupActions) > 0 ? (maxCleanupActions) : 1]; \
exceptionNode.maxCleanupAction = (maxCleanupActions) > 0 ? (maxCleanupActions) : 1; \
exceptionNode.cleanupActions = cleanupActions; \
exceptionPushNode(&exceptionNode); \
int32_t caughtException = setjmp(exceptionNode.jb); \
if (caughtException == 0)
#define CATCH(code) \
int32_t code = exceptionPopNode(); \
if (caughtException == 1)
#define FINALLY(code) int32_t code = exceptionPopNode();
#define END_TRY \
} \
while (0) \
;
#define THROW(x) exceptionThrow((x))
#define CAUGHT_EXCEPTION() ((bool)(caughtException == 1))
#define CLEANUP_EXECUTE() cleanupExecute(&exceptionNode, CAUGHT_EXCEPTION())
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_EXCEPTION_H_*/

5
packaging/tools/install.sh
View File

@ -229,6 +229,9 @@ function install_bin() {
if [ -d ${script_dir}/${xname}/bin ]; then if [ -d ${script_dir}/${xname}/bin ]; then
${csudo}cp -r ${script_dir}/${xname}/bin/* ${install_main_dir}/bin ${csudo}cp -r ${script_dir}/${xname}/bin/* ${install_main_dir}/bin
fi fi
if [ -e ${script_dir}/${xname}/uninstall_${xname}.sh ]; then
${csudo}cp -r ${script_dir}/${xname}/uninstall_${xname}.sh ${install_main_dir}/uninstall_${xname}.sh
fi
fi fi
if [ -f ${script_dir}/bin/quick_deploy.sh ]; then if [ -f ${script_dir}/bin/quick_deploy.sh ]; then
@ -250,6 +253,8 @@ function install_bin() {
for service in "${services[@]}"; do for service in "${services[@]}"; do
[ -x ${install_main_dir}/bin/${service} ] && ${csudo}ln -sf ${install_main_dir}/bin/${service} ${bin_link_dir}/${service} || : [ -x ${install_main_dir}/bin/${service} ] && ${csudo}ln -sf ${install_main_dir}/bin/${service} ${bin_link_dir}/${service} || :
done done
[ -x ${install_main_dir}/uninstall_${xname}.sh ] && ${csudo}ln -sf ${install_main_dir}/uninstall_${xname}.sh ${bin_link_dir}/uninstall_${xname}.sh || :
} }
function install_lib() { function install_lib() {

4
packaging/tools/makepkg.sh
View File

@ -363,8 +363,8 @@ if [ "$verMode" == "cluster" ]; then
# copy taosx # copy taosx
if [ -d ${top_dir}/../enterprise/src/plugins/taosx/release/taosx ]; then if [ -d ${top_dir}/../enterprise/src/plugins/taosx/release/taosx ]; then
cp -r ${top_dir}/../enterprise/src/plugins/taosx/release/taosx ${install_dir} cp -r ${top_dir}/../enterprise/src/plugins/taosx/release/taosx ${install_dir}
cp ${top_dir}/../enterprise/src/plugins/taosx/packaging/uninstall.sh ${install_dir}/taosx cp ${top_dir}/../enterprise/src/plugins/taosx/packaging/uninstall.sh ${install_dir}/taosx/uninstall_taosx.sh
sed -i 's/target=\"\"/target=\"taosx\"/g' ${install_dir}/taosx/uninstall.sh sed -i "s/uninstall.sh/uninstall_taosx.sh/g" ${install_dir}/taosx/uninstall_taosx.sh
fi fi
fi fi
fi fi

49
packaging/tools/remove.sh
View File

@ -56,7 +56,11 @@ local_bin_link_dir="/usr/local/bin"
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
config_dir="/etc/${PREFIX}" config_dir="/etc/${PREFIX}"
services=(${PREFIX}"d" ${PREFIX}"adapter" ${PREFIX}"x" ${PREFIX}"-explorer" ${PREFIX}"keeper") if [ "${verMode}" == "cluster" ]; then
services=(${PREFIX}"d" ${PREFIX}"adapter" ${PREFIX}"keeper")
else
services=(${PREFIX}"d" ${PREFIX}"adapter" ${PREFIX}"keeper" ${PREFIX}"-explorer")
fi
tools=(${PREFIX} ${PREFIX}"Benchmark" ${PREFIX}"dump" ${PREFIX}"demo" udfd set_core.sh TDinsight.sh $uninstallScript start-all.sh stop-all.sh) tools=(${PREFIX} ${PREFIX}"Benchmark" ${PREFIX}"dump" ${PREFIX}"demo" udfd set_core.sh TDinsight.sh $uninstallScript start-all.sh stop-all.sh)
csudo="" csudo=""
@ -222,6 +226,30 @@ function remove_data_and_config() {
[ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir} [ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir}
} }
echo
echo "Do you want to remove all the data, log and configuration files? [y/n]"
read answer
remove_flag=false
if [ X$answer == X"y" ] || [ X$answer == X"Y" ]; then
confirmMsg="I confirm that I would like to delete all data, log and configuration files"
echo "Please enter '${confirmMsg}' to continue"
read answer
if [ X"$answer" == X"${confirmMsg}" ]; then
remove_flag=true
else
echo "answer doesn't match, skip this step"
fi
fi
echo
if [ -e ${install_main_dir}/uninstall_${PREFIX}x.sh ]; then
if [ X$remove_flag == X"true" ]; then
bash ${install_main_dir}/uninstall_${PREFIX}x.sh --clean-all true
else
bash ${install_main_dir}/uninstall_${PREFIX}x.sh --clean-all false
fi
fi
remove_bin remove_bin
clean_header clean_header
# Remove lib file # Remove lib file
@ -232,6 +260,11 @@ clean_log
clean_config clean_config
# Remove data link directory # Remove data link directory
${csudo}rm -rf ${data_link_dir} || : ${csudo}rm -rf ${data_link_dir} || :
if [ X$remove_flag == X"true" ]; then
remove_data_and_config
fi
${csudo}rm -rf ${install_main_dir} || : ${csudo}rm -rf ${install_main_dir} || :
if [[ -e /etc/os-release ]]; then if [[ -e /etc/os-release ]]; then
osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
@ -254,20 +287,6 @@ if [ "$osType" = "Darwin" ]; then
${csudo}rm -rf /Applications/TDengine.app ${csudo}rm -rf /Applications/TDengine.app
fi fi
echo
echo "Do you want to remove all the data, log and configuration files? [y/n]"
read answer
if [ X$answer == X"y" ] || [ X$answer == X"Y" ]; then
confirmMsg="I confirm that I would like to delete all data, log and configuration files"
echo "Please enter '${confirmMsg}' to continue"
read answer
if [ X"$answer" == X"${confirmMsg}" ]; then
remove_data_and_config
else
echo "answer doesn't match, skip this step"
fi
fi
command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true
echo echo
echo "${productName} is removed successfully!" echo "${productName} is removed successfully!"

22
packaging/tools/remove_client.sh
View File

@ -31,6 +31,8 @@ bin_link_dir="/usr/bin"
lib_link_dir="/usr/lib" lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64" lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include" inc_link_dir="/usr/include"
log_dir="/var/log/${clientName2}"
cfg_dir="/etc/${clientName2}"
csudo="" csudo=""
if command -v sudo > /dev/null; then if command -v sudo > /dev/null; then
@ -92,6 +94,24 @@ function clean_log() {
${csudo}rm -rf ${log_link_dir} || : ${csudo}rm -rf ${log_link_dir} || :
} }
function clean_config_and_log_dir() {
# Remove link
echo "Do you want to remove all the log and configuration files? [y/n]"
read answer
if [ X$answer == X"y" ] || [ X$answer == X"Y" ]; then
confirmMsg="I confirm that I would like to delete all log and configuration files"
echo "Please enter '${confirmMsg}' to continue"
read answer
if [ X"$answer" == X"${confirmMsg}" ]; then
# Remove dir
rm -rf ${cfg_dir} || :
rm -rf ${log_dir} || :
else
echo "answer doesn't match, skip this step"
fi
fi
}
# Stop client. # Stop client.
kill_client kill_client
# Remove binary file and links # Remove binary file and links
@ -104,6 +124,8 @@ clean_lib
clean_log clean_log
# Remove link configuration file # Remove link configuration file
clean_config clean_config
# Remove dir
clean_config_and_log_dir
${csudo}rm -rf ${install_main_dir} ${csudo}rm -rf ${install_main_dir}

152
source/client/src/clientEnv.c
View File

@ -26,6 +26,7 @@
#include "qworker.h" #include "qworker.h"
#include "scheduler.h" #include "scheduler.h"
#include "tcache.h" #include "tcache.h"
#include "tcompare.h"
#include "tglobal.h" #include "tglobal.h"
#include "thttp.h" #include "thttp.h"
#include "tmsg.h" #include "tmsg.h"
@ -35,7 +36,6 @@
#include "tsched.h" #include "tsched.h"
#include "ttime.h" #include "ttime.h"
#include "tversion.h" #include "tversion.h"
#include "tcompare.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h" #include "cus_name.h"
@ -44,16 +44,16 @@
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
#define ENV_JSON_FALSE_CHECK(c) \ #define ENV_JSON_FALSE_CHECK(c) \
do { \ do { \
if (!c) { \ if (!c) { \
tscError("faild to add item to JSON object");\ tscError("faild to add item to JSON object"); \
code = TSDB_CODE_TSC_FAIL_GENERATE_JSON; \ code = TSDB_CODE_TSC_FAIL_GENERATE_JSON; \
goto _end; \ goto _end; \
} \ } \
} while (0) } while (0)
#define ENV_ERR_RET(c,info) \ #define ENV_ERR_RET(c, info) \
do { \ do { \
int32_t _code = c; \ int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \ if (_code != TSDB_CODE_SUCCESS) { \
@ -94,109 +94,110 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) {
int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1); int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1);
int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, ", current:%d, app current:%d, total:%d, QID:0x%" PRIx64,
pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
} }
return code; return code;
} }
static void concatStrings(SArray *list, char* buf, int size){ static void concatStrings(SArray *list, char *buf, int size) {
int len = 0; int len = 0;
for(int i = 0; i < taosArrayGetSize(list); i++){ for (int i = 0; i < taosArrayGetSize(list); i++) {
char* db = taosArrayGet(list, i); char *db = taosArrayGet(list, i);
if (NULL == db) { if (NULL == db) {
tscError("get dbname failed, buf:%s", buf); tscError("get dbname failed, buf:%s", buf);
break; break;
} }
char* dot = strchr(db, '.'); char *dot = strchr(db, '.');
if (dot != NULL) { if (dot != NULL) {
db = dot + 1; db = dot + 1;
} }
if (i != 0){ if (i != 0) {
(void)strcat(buf, ","); (void)strcat(buf, ",");
len += 1; len += 1;
} }
int ret = snprintf(buf + len, size - len, "%s", db); int ret = snprintf(buf + len, size - len, "%s", db);
if (ret < 0) { if (ret < 0) {
tscError("snprintf failed, buf:%s, ret:%d", buf, ret); tscError("snprintf failed, buf:%s, ret:%d", buf, ret);
break; break;
} }
len += ret; len += ret;
if (len >= size){ if (len >= size) {
tscInfo("dbList is truncated, buf:%s, len:%d", buf, len); tscInfo("dbList is truncated, buf:%s, len:%d", buf, len);
break; break;
} }
} }
} }
static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration){ static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration) {
cJSON* json = cJSON_CreateObject(); cJSON *json = cJSON_CreateObject();
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (json == NULL) { if (json == NULL) {
tscError("[monitor] cJSON_CreateObject failed"); tscError("[monitor] cJSON_CreateObject failed");
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
char clusterId[32] = {0}; char clusterId[32] = {0};
if (snprintf(clusterId, sizeof(clusterId), "%" PRId64, pTscObj->pAppInfo->clusterId) < 0){ if (snprintf(clusterId, sizeof(clusterId), "%" PRId64, pTscObj->pAppInfo->clusterId) < 0) {
tscError("failed to generate clusterId:%" PRId64, pTscObj->pAppInfo->clusterId); tscError("failed to generate clusterId:%" PRId64, pTscObj->pAppInfo->clusterId);
code = TSDB_CODE_FAILED; code = TSDB_CODE_FAILED;
goto _end; goto _end;
} }
char startTs[32] = {0}; char startTs[32] = {0};
if (snprintf(startTs, sizeof(startTs), "%" PRId64, pRequest->metric.start/1000) < 0){ if (snprintf(startTs, sizeof(startTs), "%" PRId64, pRequest->metric.start / 1000) < 0) {
tscError("failed to generate startTs:%" PRId64, pRequest->metric.start/1000); tscError("failed to generate startTs:%" PRId64, pRequest->metric.start / 1000);
code = TSDB_CODE_FAILED; code = TSDB_CODE_FAILED;
goto _end; goto _end;
} }
char requestId[32] = {0}; char requestId[32] = {0};
if (snprintf(requestId, sizeof(requestId), "%" PRIu64, pRequest->requestId) < 0){ if (snprintf(requestId, sizeof(requestId), "%" PRIu64, pRequest->requestId) < 0) {
tscError("failed to generate requestId:%" PRIu64, pRequest->requestId); tscError("failed to generate requestId:%" PRIu64, pRequest->requestId);
code = TSDB_CODE_FAILED; code = TSDB_CODE_FAILED;
goto _end; goto _end;
} }
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "cluster_id", cJSON_CreateString(clusterId))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "cluster_id", cJSON_CreateString(clusterId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "start_ts", cJSON_CreateString(startTs))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "start_ts", cJSON_CreateString(startTs)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "request_id", cJSON_CreateString(requestId))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "request_id", cJSON_CreateString(requestId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "query_time", cJSON_CreateNumber(duration/1000))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "query_time", cJSON_CreateNumber(duration / 1000)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "code", cJSON_CreateNumber(pRequest->code))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "code", cJSON_CreateNumber(pRequest->code)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code)))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code))));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(
if(pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){ json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows)));
if (pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen) {
char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen]; char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen];
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0'; pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0';
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = tmp; pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = tmp;
}else{ } else {
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
} }
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "user", cJSON_CreateString(pTscObj->user))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "user", cJSON_CreateString(pTscObj->user)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_name", cJSON_CreateString(appInfo.appName))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_name", cJSON_CreateString(appInfo.appName)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "ip", cJSON_CreateString(tsLocalFqdn))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "ip", cJSON_CreateString(tsLocalFqdn)));
char pid[32] = {0}; char pid[32] = {0};
if (snprintf(pid, sizeof(pid), "%d", appInfo.pid) < 0){ if (snprintf(pid, sizeof(pid), "%d", appInfo.pid) < 0) {
tscError("failed to generate pid:%d", appInfo.pid); tscError("failed to generate pid:%d", appInfo.pid);
code = TSDB_CODE_FAILED; code = TSDB_CODE_FAILED;
goto _end; goto _end;
} }
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_id", cJSON_CreateString(pid))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_id", cJSON_CreateString(pid)));
if(pRequest->dbList != NULL){ if (pRequest->dbList != NULL) {
char dbList[1024] = {0}; char dbList[1024] = {0};
concatStrings(pRequest->dbList, dbList, sizeof(dbList) - 1); concatStrings(pRequest->dbList, dbList, sizeof(dbList) - 1);
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(dbList))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(dbList)));
}else if(pRequest->pDb != NULL){ } else if (pRequest->pDb != NULL) {
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(pRequest->pDb))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(pRequest->pDb)));
}else{ } else {
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(""))); ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString("")));
} }
char* value = cJSON_PrintUnformatted(json); char *value = cJSON_PrintUnformatted(json);
MonitorSlowLogData data = {0}; MonitorSlowLogData data = {0};
data.clusterId = pTscObj->pAppInfo->clusterId; data.clusterId = pTscObj->pAppInfo->clusterId;
data.type = SLOW_LOG_WRITE; data.type = SLOW_LOG_WRITE;
@ -212,7 +213,7 @@ _end:
return code; return code;
} }
static bool checkSlowLogExceptDb(SRequestObj *pRequest, char* exceptDb) { static bool checkSlowLogExceptDb(SRequestObj *pRequest, char *exceptDb) {
if (pRequest->pDb != NULL) { if (pRequest->pDb != NULL) {
return strcmp(pRequest->pDb, exceptDb) != 0; return strcmp(pRequest->pDb, exceptDb) != 0;
} }
@ -227,7 +228,7 @@ static bool checkSlowLogExceptDb(SRequestObj *pRequest, char* exceptDb) {
if (dot != NULL) { if (dot != NULL) {
db = dot + 1; db = dot + 1;
} }
if(strcmp(db, exceptDb) == 0){ if (strcmp(db, exceptDb) == 0) {
return false; return false;
} }
} }
@ -248,15 +249,14 @@ static void deregisterRequest(SRequestObj *pRequest) {
int32_t reqType = SLOW_LOG_TYPE_OTHERS; int32_t reqType = SLOW_LOG_TYPE_OTHERS;
int64_t duration = taosGetTimestampUs() - pRequest->metric.start; int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", QID:0x%" PRIx64
" elapsed:%.2f ms, " " elapsed:%.2f ms, "
"current:%d, app current:%d", "current:%d, app current:%d",
pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst);
if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) { if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) {
if ((pRequest->pQuery && pRequest->pQuery->pRoot && if ((pRequest->pQuery && pRequest->pQuery->pRoot && QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type &&
QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type && (0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) ||
(0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) ||
QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType) { QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType) {
tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64
"us, planCost:%" PRId64 "us, exec:%" PRId64 "us", "us, planCost:%" PRId64 "us, exec:%" PRId64 "us",
@ -279,7 +279,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
} }
} }
if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){ if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) {
if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) { if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT); sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT);
} else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
@ -289,14 +289,15 @@ static void deregisterRequest(SRequestObj *pRequest) {
} }
} }
if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL || duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) && if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL ||
duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) &&
checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->monitorParas.tsSlowLogExceptDb)) { checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->monitorParas.tsSlowLogExceptDb)) {
(void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); (void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1);
if (pTscObj->pAppInfo->monitorParas.tsSlowLogScope & reqType) { if (pTscObj->pAppInfo->monitorParas.tsSlowLogScope & reqType) {
taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 " us, Duration:%" PRId64 "us, SQL:%s", taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 " us, Duration:%" PRId64 "us, SQL:%s",
taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration,
pRequest->sqlstr); pRequest->sqlstr);
if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){ if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) {
slowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration); slowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration);
if (TSDB_CODE_SUCCESS != generateWriteSlowLog(pTscObj, pRequest, reqType, duration)) { if (TSDB_CODE_SUCCESS != generateWriteSlowLog(pTscObj, pRequest, reqType, duration)) {
tscError("failed to generate write slow log"); tscError("failed to generate write slow log");
@ -389,7 +390,7 @@ void destroyAllRequests(SHashObj *pRequests) {
SRequestObj *pRequest = acquireRequest(*rid); SRequestObj *pRequest = acquireRequest(*rid);
if (pRequest) { if (pRequest) {
destroyRequest(pRequest); destroyRequest(pRequest);
(void)releaseRequest(*rid); // ignore error (void)releaseRequest(*rid); // ignore error
} }
pIter = taosHashIterate(pRequests, pIter); pIter = taosHashIterate(pRequests, pIter);
@ -404,7 +405,7 @@ void stopAllRequests(SHashObj *pRequests) {
SRequestObj *pRequest = acquireRequest(*rid); SRequestObj *pRequest = acquireRequest(*rid);
if (pRequest) { if (pRequest) {
taos_stop_query(pRequest); taos_stop_query(pRequest);
(void)releaseRequest(*rid); // ignore error (void)releaseRequest(*rid); // ignore error
} }
pIter = taosHashIterate(pRequests, pIter); pIter = taosHashIterate(pRequests, pIter);
@ -412,7 +413,7 @@ void stopAllRequests(SHashObj *pRequests) {
} }
void destroyAppInst(void *info) { void destroyAppInst(void *info) {
SAppInstInfo* pAppInfo = *(SAppInstInfo**)info; SAppInstInfo *pAppInfo = *(SAppInstInfo **)info;
tscDebug("destroy app inst mgr %p", pAppInfo); tscDebug("destroy app inst mgr %p", pAppInfo);
int32_t code = taosThreadMutexLock(&appInfo.mutex); int32_t code = taosThreadMutexLock(&appInfo.mutex);
@ -597,28 +598,28 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri
int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
/// return the most previous req ref id /// return the most previous req ref id
int64_t removeFromMostPrevReq(SRequestObj* pRequest) { int64_t removeFromMostPrevReq(SRequestObj *pRequest) {
int64_t mostPrevReqRefId = pRequest->self; int64_t mostPrevReqRefId = pRequest->self;
SRequestObj* pTmp = pRequest; SRequestObj *pTmp = pRequest;
while (pTmp->relation.prevRefId) { while (pTmp->relation.prevRefId) {
pTmp = acquireRequest(pTmp->relation.prevRefId); pTmp = acquireRequest(pTmp->relation.prevRefId);
if (pTmp) { if (pTmp) {
mostPrevReqRefId = pTmp->self; mostPrevReqRefId = pTmp->self;
(void)releaseRequest(mostPrevReqRefId); // ignore error (void)releaseRequest(mostPrevReqRefId); // ignore error
} else { } else {
break; break;
} }
} }
(void)removeRequest(mostPrevReqRefId); // ignore error (void)removeRequest(mostPrevReqRefId); // ignore error
return mostPrevReqRefId; return mostPrevReqRefId;
} }
void destroyNextReq(int64_t nextRefId) { void destroyNextReq(int64_t nextRefId) {
if (nextRefId) { if (nextRefId) {
SRequestObj* pObj = acquireRequest(nextRefId); SRequestObj *pObj = acquireRequest(nextRefId);
if (pObj) { if (pObj) {
(void)releaseRequest(nextRefId); // ignore error (void)releaseRequest(nextRefId); // ignore error
(void)releaseRequest(nextRefId); // ignore error (void)releaseRequest(nextRefId); // ignore error
} }
} }
} }
@ -638,7 +639,7 @@ void destroySubRequests(SRequestObj *pRequest) {
pTmp = acquireRequest(tmpRefId); pTmp = acquireRequest(tmpRefId);
if (pTmp) { if (pTmp) {
pReqList[++reqIdx] = pTmp; pReqList[++reqIdx] = pTmp;
(void)releaseRequest(tmpRefId); // ignore error (void)releaseRequest(tmpRefId); // ignore error
} else { } else {
tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId); tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId);
break; break;
@ -646,7 +647,7 @@ void destroySubRequests(SRequestObj *pRequest) {
} }
for (int32_t i = reqIdx; i >= 0; i--) { for (int32_t i = reqIdx; i >= 0; i--) {
(void)removeRequest(pReqList[i]->self); // ignore error (void)removeRequest(pReqList[i]->self); // ignore error
} }
tmpRefId = pRequest->relation.nextRefId; tmpRefId = pRequest->relation.nextRefId;
@ -654,8 +655,8 @@ void destroySubRequests(SRequestObj *pRequest) {
pTmp = acquireRequest(tmpRefId); pTmp = acquireRequest(tmpRefId);
if (pTmp) { if (pTmp) {
tmpRefId = pTmp->relation.nextRefId; tmpRefId = pTmp->relation.nextRefId;
(void)removeRequest(pTmp->self); // ignore error (void)removeRequest(pTmp->self); // ignore error
(void)releaseRequest(pTmp->self); // ignore error (void)releaseRequest(pTmp->self); // ignore error
} else { } else {
tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId);
break; break;
@ -758,7 +759,7 @@ void stopAllQueries(SRequestObj *pRequest) {
for (int32_t i = reqIdx; i >= 0; i--) { for (int32_t i = reqIdx; i >= 0; i--) {
taosStopQueryImpl(pReqList[i]); taosStopQueryImpl(pReqList[i]);
(void)releaseRequest(pReqList[i]->self); // ignore error (void)releaseRequest(pReqList[i]->self); // ignore error
} }
taosStopQueryImpl(pRequest); taosStopQueryImpl(pRequest);
@ -769,7 +770,7 @@ void stopAllQueries(SRequestObj *pRequest) {
if (pTmp) { if (pTmp) {
tmpRefId = pTmp->relation.nextRefId; tmpRefId = pTmp->relation.nextRefId;
taosStopQueryImpl(pTmp); taosStopQueryImpl(pTmp);
(void)releaseRequest(pTmp->self); // ignore error (void)releaseRequest(pTmp->self); // ignore error
} else { } else {
tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId);
break; break;
@ -929,7 +930,8 @@ void taos_init_imp(void) {
appInfo.pid = taosGetPId(); appInfo.pid = taosGetPId();
appInfo.startTime = taosGetTimestampMs(); appInfo.startTime = taosGetTimestampMs();
appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
appInfo.pInstMapByClusterId = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); appInfo.pInstMapByClusterId =
taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
if (NULL == appInfo.pInstMap || NULL == appInfo.pInstMapByClusterId) { if (NULL == appInfo.pInstMap || NULL == appInfo.pInstMapByClusterId) {
tscError("failed to allocate memory when init appInfo"); tscError("failed to allocate memory when init appInfo");
tscInitRes = TSDB_CODE_OUT_OF_MEMORY; tscInitRes = TSDB_CODE_OUT_OF_MEMORY;
@ -993,8 +995,8 @@ int taos_init() {
return tscInitRes; return tscInitRes;
} }
const char* getCfgName(TSDB_OPTION option) { const char *getCfgName(TSDB_OPTION option) {
const char* name = NULL; const char *name = NULL;
switch (option) { switch (option) {
case TSDB_OPTION_SHELL_ACTIVITY_TIMER: case TSDB_OPTION_SHELL_ACTIVITY_TIMER:
@ -1025,12 +1027,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
char newstr[PATH_MAX]; char newstr[PATH_MAX];
int len = strlen(str); int len = strlen(str);
if (len > 1 && str[0] != '"' && str[0] != '\'') { if (len > 1 && str[0] != '"' && str[0] != '\'') {
if (len + 2 >= PATH_MAX) { if (len + 2 >= PATH_MAX) {
tscError("Too long path %s", str); tscError("Too long path %s", str);
return -1; return -1;
} }
newstr[0] = '"'; newstr[0] = '"';
(void)memcpy(newstr+1, str, len); (void)memcpy(newstr + 1, str, len);
newstr[len + 1] = '"'; newstr[len + 1] = '"';
newstr[len + 2] = '\0'; newstr[len + 2] = '\0';
str = newstr; str = newstr;

117
source/client/src/clientImpl.c
View File

@ -15,8 +15,8 @@
#include "cJSON.h" #include "cJSON.h"
#include "clientInt.h" #include "clientInt.h"
#include "clientMonitor.h"
#include "clientLog.h" #include "clientLog.h"
#include "clientMonitor.h"
#include "command.h" #include "command.h"
#include "scheduler.h" #include "scheduler.h"
#include "tdatablock.h" #include "tdatablock.h"
@ -28,7 +28,7 @@
#include "tref.h" #include "tref.h"
#include "tsched.h" #include "tsched.h"
#include "tversion.h" #include "tversion.h"
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo); static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo);
static bool stringLengthCheck(const char* str, size_t maxsize) { static bool stringLengthCheck(const char* str, size_t maxsize) {
@ -68,15 +68,13 @@ bool chkRequestKilled(void* param) {
return killed; return killed;
} }
void cleanupAppInfo() { void cleanupAppInfo() { taosHashCleanup(appInfo.pInstMap); }
taosHashCleanup(appInfo.pInstMap);
}
static int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, static int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj); SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj);
int32_t taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, int32_t taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port, int connType, STscObj** pObj) { uint16_t port, int connType, STscObj** pObj) {
TSC_ERR_RET(taos_init()); TSC_ERR_RET(taos_init());
if (!validateUserName(user)) { if (!validateUserName(user)) {
TSC_ERR_RET(TSDB_CODE_TSC_INVALID_USER_LENGTH); TSC_ERR_RET(TSDB_CODE_TSC_INVALID_USER_LENGTH);
@ -126,7 +124,7 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass
} }
SAppInstInfo** pInst = NULL; SAppInstInfo** pInst = NULL;
int32_t code = taosThreadMutexLock(&appInfo.mutex); int32_t code = taosThreadMutexLock(&appInfo.mutex);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
tscError("failed to lock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code))); tscError("failed to lock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code)));
TSC_ERR_RET(code); TSC_ERR_RET(code);
@ -176,25 +174,29 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass
_return: _return:
code = taosThreadMutexUnlock(&appInfo.mutex);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
tscError("failed to unlock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code))); (void)taosThreadMutexUnlock(&appInfo.mutex);
taosMemoryFreeClear(key);
return code; return code;
} else {
code = taosThreadMutexUnlock(&appInfo.mutex);
taosMemoryFreeClear(key);
if (TSDB_CODE_SUCCESS != code) {
tscError("failed to unlock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code)));
return code;
}
return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType, pObj);
} }
taosMemoryFreeClear(key);
return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType, pObj);
} }
//SAppInstInfo* getAppInstInfo(const char* clusterKey) { // SAppInstInfo* getAppInstInfo(const char* clusterKey) {
// SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey)); // SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
// if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) { // if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
// return *ppAppInstInfo; // return *ppAppInstInfo;
// } else { // } else {
// return NULL; // return NULL;
// } // }
//} // }
void freeQueryParam(SSyncQueryParam* param) { void freeQueryParam(SSyncQueryParam* param) {
if (param == NULL) return; if (param == NULL) return;
@ -230,7 +232,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
sizeof((*pRequest)->self)); sizeof((*pRequest)->self));
if (err) { if (err) {
tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", tscError("%" PRId64 " failed to add to request container, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
destroyRequest(*pRequest); destroyRequest(*pRequest);
*pRequest = NULL; *pRequest = NULL;
@ -241,7 +243,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) { if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) {
if (TSDB_CODE_SUCCESS != if (TSDB_CODE_SUCCESS !=
nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", tscError("%" PRId64 " failed to create node allocator, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
destroyRequest(*pRequest); destroyRequest(*pRequest);
*pRequest = NULL; *pRequest = NULL;
@ -249,7 +251,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
} }
} }
tscDebugL("0x%" PRIx64 " SQL: %s, reqId:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId); tscDebugL("0x%" PRIx64 " SQL: %s, QID:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -363,10 +365,10 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0; pResultInfo->numOfRows = 0;
tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId); pRequest->requestId);
} else { } else {
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64,
pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
pRequest->requestId); pRequest->requestId);
} }
@ -432,7 +434,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qnodeRequired(SRequestObj* pRequest, bool *required) { int32_t qnodeRequired(SRequestObj* pRequest, bool* required) {
if (QUERY_POLICY_VNODE == tsQueryPolicy || QUERY_POLICY_CLIENT == tsQueryPolicy) { if (QUERY_POLICY_VNODE == tsQueryPolicy || QUERY_POLICY_CLIENT == tsQueryPolicy) {
*required = false; *required = false;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -554,7 +556,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr
if (NULL == nodeList) { if (NULL == nodeList) {
return terrno; return terrno;
} }
char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client"; char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client";
int32_t dbNum = taosArrayGetSize(pDbVgList); int32_t dbNum = taosArrayGetSize(pDbVgList);
for (int32_t i = 0; i < dbNum; ++i) { for (int32_t i = 0; i < dbNum; ++i) {
@ -568,7 +570,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr
} }
for (int32_t j = 0; j < vgNum; ++j) { for (int32_t j = 0; j < vgNum; ++j) {
SVgroupInfo* pInfo = taosArrayGet(pVg, j); SVgroupInfo* pInfo = taosArrayGet(pVg, j);
if (NULL == pInfo) { if (NULL == pInfo) {
taosArrayDestroy(nodeList); taosArrayDestroy(nodeList);
return TSDB_CODE_OUT_OF_RANGE; return TSDB_CODE_OUT_OF_RANGE;
@ -972,8 +974,8 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
break; break;
} }
default: default:
tscError("0x%" PRIx64 ", invalid exec result for request type %d, reqId:0x%" PRIx64, pRequest->self, tscError("0x%" PRIx64 ", invalid exec result for request type %d, QID:0x%" PRIx64, pRequest->self, pRequest->type,
pRequest->type, pRequest->requestId); pRequest->requestId);
code = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
} }
@ -1017,12 +1019,12 @@ void returnToUser(SRequestObj* pRequest) {
(void)releaseRequest(pRequest->relation.userRefId); (void)releaseRequest(pRequest->relation.userRefId);
return; return;
} else { } else {
tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self,
pRequest->relation.userRefId, pRequest->requestId); pRequest->relation.userRefId, pRequest->requestId);
} }
} }
static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock**pBlock) { static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock** pBlock) {
int64_t lastTs = 0; int64_t lastTs = 0;
TAOS_FIELD* pResFields = taos_fetch_fields(pRes); TAOS_FIELD* pResFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes); int32_t numOfFields = taos_num_fields(pRes);
@ -1032,7 +1034,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
return code; return code;
} }
for(int32_t i = 0; i < numOfFields; ++i) { for (int32_t i = 0; i < numOfFields; ++i) {
SColumnInfoData colInfoData = createColumnInfoData(pResFields[i].type, pResFields[i].bytes, i + 1); SColumnInfoData colInfoData = createColumnInfoData(pResFields[i].type, pResFields[i].bytes, i + 1);
code = blockDataAppendColInfo(*pBlock, &colInfoData); code = blockDataAppendColInfo(*pBlock, &colInfoData);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
@ -1054,7 +1056,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
lastTs = ts; lastTs = ts;
} }
for(int32_t j = 0; j < numOfFields; ++j) { for (int32_t j = 0; j < numOfFields; ++j) {
SColumnInfoData* pColInfoData = taosArrayGet((*pBlock)->pDataBlock, j); SColumnInfoData* pColInfoData = taosArrayGet((*pBlock)->pDataBlock, j);
code = colDataSetVal(pColInfoData, i, pRow[j], false); code = colDataSetVal(pColInfoData, i, pRow[j], false);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
@ -1069,7 +1071,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
(*pBlock)->info.window.ekey = lastTs; (*pBlock)->info.window.ekey = lastTs;
(*pBlock)->info.rows = numOfRows; (*pBlock)->info.rows = numOfRows;
tscDebug("lastKey:%"PRId64" numOfRows:%d from all vgroups", lastTs, numOfRows); tscDebug("lastKey:%" PRId64 " numOfRows:%d from all vgroups", lastTs, numOfRows);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1082,7 +1084,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
SSDataBlock* pBlock = NULL; SSDataBlock* pBlock = NULL;
if (TSDB_CODE_SUCCESS != createResultBlock(res, rowNum, &pBlock)) { if (TSDB_CODE_SUCCESS != createResultBlock(res, rowNum, &pBlock)) {
tscError("0x%" PRIx64 ", create result block failed, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); tscError("0x%" PRIx64 ", create result block failed, QID:0x%" PRIx64, pRequest->self, pRequest->requestId);
return; return;
} }
@ -1091,7 +1093,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
continuePostSubQuery(pNextReq, pBlock); continuePostSubQuery(pNextReq, pBlock);
(void)releaseRequest(pRequest->relation.nextRefId); (void)releaseRequest(pRequest->relation.nextRefId);
} else { } else {
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self,
pRequest->relation.nextRefId, pRequest->requestId); pRequest->relation.nextRefId, pRequest->requestId);
} }
@ -1110,7 +1112,7 @@ void handlePostSubQuery(SSqlCallbackWrapper* pWrapper) {
continuePostSubQuery(pNextReq, NULL); continuePostSubQuery(pNextReq, NULL);
(void)releaseRequest(pRequest->relation.nextRefId); (void)releaseRequest(pRequest->relation.nextRefId);
} else { } else {
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self, tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self,
pRequest->relation.nextRefId, pRequest->requestId); pRequest->relation.nextRefId, pRequest->requestId);
} }
} }
@ -1141,11 +1143,11 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
} }
taosMemoryFree(pResult); taosMemoryFree(pResult);
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId); pRequest->requestId);
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) { if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, QID:0x%" PRIx64, pRequest->self,
tstrerror(code), pRequest->retry, pRequest->requestId); tstrerror(code), pRequest->retry, pRequest->requestId);
(void)removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type)); (void)removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
restartAsyncQuery(pRequest, code); restartAsyncQuery(pRequest, code);
@ -1215,7 +1217,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
} }
break; break;
case QUERY_EXEC_MODE_SCHEDULE: { case QUERY_EXEC_MODE_SCHEDULE: {
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
if (NULL == pMnodeList) { if (NULL == pMnodeList) {
code = terrno; code = terrno;
break; break;
@ -1533,7 +1535,7 @@ int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* p
} }
int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj) { SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj) {
*pTscObj = NULL; *pTscObj = NULL;
int32_t code = createTscObj(user, auth, db, connType, pAppInfo, pTscObj); int32_t code = createTscObj(user, auth, db, connType, pAppInfo, pTscObj);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
@ -1560,7 +1562,8 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta
} }
int64_t transporterId = 0; int64_t transporterId = 0;
code = asyncSendMsgToServer((*pTscObj)->pAppInfo->pTransporter, &(*pTscObj)->pAppInfo->mgmtEp.epSet, &transporterId, body); code = asyncSendMsgToServer((*pTscObj)->pAppInfo->pTransporter, &(*pTscObj)->pAppInfo->mgmtEp.epSet, &transporterId,
body);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
destroyTscObj(*pTscObj); destroyTscObj(*pTscObj);
tscError("failed to send connect msg to server, code:%s", tstrerror(code)); tscError("failed to send connect msg to server, code:%s", tstrerror(code));
@ -1577,7 +1580,7 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta
*pTscObj = NULL; *pTscObj = NULL;
return terrno; return terrno;
} else { } else {
tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, (*pTscObj)->id, tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, QID:0x%" PRIx64, (*pTscObj)->id,
(*pTscObj)->connId, (*pTscObj)->pAppInfo->pTransporter, pRequest->requestId); (*pTscObj)->connId, (*pTscObj)->pAppInfo->pTransporter, pRequest->requestId);
destroyRequest(pRequest); destroyRequest(pRequest);
} }
@ -1708,7 +1711,7 @@ int32_t doProcessMsgFromServer(void* param) {
char tbuf[40] = {0}; char tbuf[40] = {0};
TRACE_TO_STR(trace, tbuf); TRACE_TO_STR(trace, tbuf);
tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, gtid: %s", pMsg->info.handle, tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, QID:%s", pMsg->info.handle,
TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf); TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf);
if (pSendInfo->requestObjRefId != 0) { if (pSendInfo->requestObjRefId != 0) {
@ -1792,6 +1795,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
AsyncArg* arg = taosMemoryCalloc(1, sizeof(AsyncArg)); AsyncArg* arg = taosMemoryCalloc(1, sizeof(AsyncArg));
if (NULL == arg) { if (NULL == arg) {
pMsg->code = TSDB_CODE_OUT_OF_MEMORY; pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(tEpSet);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
destroySendMsgInfo(pMsg->info.ahandle); destroySendMsgInfo(pMsg->info.ahandle);
return; return;
@ -1820,7 +1824,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
} }
STscObj* pObj = NULL; STscObj* pObj = NULL;
int32_t code = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY, &pObj); int32_t code = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY, &pObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
int64_t* rid = taosMemoryCalloc(1, sizeof(int64_t)); int64_t* rid = taosMemoryCalloc(1, sizeof(int64_t));
if (NULL == rid) { if (NULL == rid) {
@ -1890,7 +1894,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
} }
SReqResultInfo* pResInfo = &pRequest->body.resInfo; SReqResultInfo* pResInfo = &pRequest->body.resInfo;
SSchedulerReq req = { .syncReq = true, .pFetchRes = (void**)&pResInfo->pData }; SSchedulerReq req = {.syncReq = true, .pFetchRes = (void**)&pResInfo->pData};
pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req); pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
@ -1905,7 +1909,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
return NULL; return NULL;
} }
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64,
pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId);
STscObj* pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
@ -2033,7 +2037,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) {
} }
static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
char* p = (char*)pResultInfo->pData; char* p = (char*)pResultInfo->pData;
int32_t blockVersion = *(int32_t*)p; int32_t blockVersion = *(int32_t*)p;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column
@ -2298,7 +2302,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
char* pStart = p; char* pStart = p;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
if(blockVersion == BLOCK_VERSION_1){ if (blockVersion == BLOCK_VERSION_1) {
colLength[i] = htonl(colLength[i]); colLength[i] = htonl(colLength[i]);
} }
if (colLength[i] >= dataLen) { if (colLength[i] >= dataLen) {
@ -2733,7 +2737,8 @@ void syncQueryFn(void* param, void* res, int32_t code) {
(void)tsem_post(&pParam->sem); (void)tsem_post(&pParam->sem);
} }
void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) { void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly,
int8_t source) {
if (sql == NULL || NULL == fp) { if (sql == NULL || NULL == fp) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
if (fp) { if (fp) {
@ -2854,7 +2859,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
SReqResultInfo* pResultInfo = &pRequest->body.resInfo; SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code,
tstrerror(code), pRequest->requestId); tstrerror(code), pRequest->requestId);
pResultInfo->pData = pResult; pResultInfo->pData = pResult;
@ -2877,10 +2882,10 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4); setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0; pResultInfo->numOfRows = 0;
tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code), tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code),
pRequest->requestId); pRequest->requestId);
} else { } else {
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64,
pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
pRequest->requestId); pRequest->requestId);
@ -2932,7 +2937,7 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param
int32_t code = schedulerFetchRows(pRequest->body.queryJob, &req); int32_t code = schedulerFetchRows(pRequest->body.queryJob, &req);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " failed to schedule fetch rows", pRequest->requestId); tscError("0x%" PRIx64 " failed to schedule fetch rows", pRequest->requestId);
pRequest->body.fetchFp(param, pRequest, code); pRequest->body.fetchFp(param, pRequest, code);
} }
} }

97
source/client/src/clientMain.c
View File

@ -16,19 +16,19 @@
#include "catalog.h" #include "catalog.h"
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h" #include "clientLog.h"
#include "clientStmt.h"
#include "clientMonitor.h" #include "clientMonitor.h"
#include "clientStmt.h"
#include "functionMgt.h" #include "functionMgt.h"
#include "os.h" #include "os.h"
#include "query.h" #include "query.h"
#include "scheduler.h" #include "scheduler.h"
#include "tcompare.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h" #include "tmsg.h"
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
#include "version.h" #include "version.h"
#include "tcompare.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
@ -120,7 +120,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
} }
STscObj *pObj = NULL; STscObj *pObj = NULL;
int32_t code = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY, &pObj); int32_t code = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY, &pObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t)); int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t));
if (NULL == rid) { if (NULL == rid) {
@ -183,15 +183,15 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type)
return 0; return 0;
} }
typedef struct SFetchWhiteListInfo{ typedef struct SFetchWhiteListInfo {
int64_t connId; int64_t connId;
__taos_async_whitelist_fn_t userCbFn; __taos_async_whitelist_fn_t userCbFn;
void* userParam; void *userParam;
} SFetchWhiteListInfo; } SFetchWhiteListInfo;
int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { int32_t fetchWhiteListCallbackFn(void *param, SDataBuf *pMsg, int32_t code) {
SFetchWhiteListInfo* pInfo = (SFetchWhiteListInfo*)param; SFetchWhiteListInfo *pInfo = (SFetchWhiteListInfo *)param;
TAOS* taos = &pInfo->connId; TAOS *taos = &pInfo->connId;
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pInfo->userCbFn(pInfo->userParam, code, taos, 0, NULL); pInfo->userCbFn(pInfo->userParam, code, taos, 0, NULL);
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
@ -209,7 +209,7 @@ int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
return terrno; return terrno;
} }
uint64_t* pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t)); uint64_t *pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t));
if (pWhiteLists == NULL) { if (pWhiteLists == NULL) {
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
@ -238,7 +238,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
return; return;
} }
int64_t connId = *(int64_t*)taos; int64_t connId = *(int64_t *)taos;
STscObj *pTsc = acquireTscObj(connId); STscObj *pTsc = acquireTscObj(connId);
if (NULL == pTsc) { if (NULL == pTsc) {
@ -255,7 +255,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
return; return;
} }
void* pReq = taosMemoryMalloc(msgLen); void *pReq = taosMemoryMalloc(msgLen);
if (pReq == NULL) { if (pReq == NULL) {
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
releaseTscObj(connId); releaseTscObj(connId);
@ -269,7 +269,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
return; return;
} }
SFetchWhiteListInfo* pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo)); SFetchWhiteListInfo *pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo));
if (pParam == NULL) { if (pParam == NULL) {
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
taosMemoryFree(pReq); taosMemoryFree(pReq);
@ -280,9 +280,9 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
pParam->connId = connId; pParam->connId = connId;
pParam->userCbFn = fp; pParam->userCbFn = fp;
pParam->userParam = param; pParam->userParam = param;
SMsgSendInfo* pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); SMsgSendInfo *pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (pSendInfo == NULL) { if (pSendInfo == NULL) {
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
taosMemoryFree(pParam); taosMemoryFree(pParam);
taosMemoryFree(pReq); taosMemoryFree(pReq);
releaseTscObj(connId); releaseTscObj(connId);
@ -297,7 +297,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
pSendInfo->msgType = TDMT_MND_GET_USER_WHITELIST; pSendInfo->msgType = TDMT_MND_GET_USER_WHITELIST;
int64_t transportId = 0; int64_t transportId = 0;
SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp); SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp);
if (TSDB_CODE_SUCCESS != asyncSendMsgToServer(pTsc->pAppInfo->pTransporter, &epSet, &transportId, pSendInfo)) { if (TSDB_CODE_SUCCESS != asyncSendMsgToServer(pTsc->pAppInfo->pTransporter, &epSet, &transportId, pSendInfo)) {
tscWarn("failed to async send msg to server"); tscWarn("failed to async send msg to server");
} }
@ -443,7 +443,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
return NULL; return NULL;
} }
if(pRequest->inCallback) { if (pRequest->inCallback) {
tscError("can not call taos_fetch_row before query callback ends."); tscError("can not call taos_fetch_row before query callback ends.");
terrno = TSDB_CODE_TSC_INVALID_OPERATION; terrno = TSDB_CODE_TSC_INVALID_OPERATION;
return NULL; return NULL;
@ -454,7 +454,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
SMqRspObj *msg = ((SMqRspObj *)res); SMqRspObj *msg = ((SMqRspObj *)res);
SReqResultInfo *pResultInfo = NULL; SReqResultInfo *pResultInfo = NULL;
if (msg->common.resIter == -1) { if (msg->common.resIter == -1) {
if(tmqGetNextResInfo(res, true, &pResultInfo) != 0){ if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) {
return NULL; return NULL;
} }
} else { } else {
@ -466,7 +466,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
pResultInfo->current += 1; pResultInfo->current += 1;
return pResultInfo->row; return pResultInfo->row;
} else { } else {
if (tmqGetNextResInfo(res, true, &pResultInfo) != 0){ if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) {
return NULL; return NULL;
} }
@ -540,22 +540,23 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
len += sprintf(str + len, "%lf", dv); len += sprintf(str + len, "%lf", dv);
} break; } break;
case TSDB_DATA_TYPE_VARBINARY:{ case TSDB_DATA_TYPE_VARBINARY: {
void* data = NULL; void *data = NULL;
uint32_t size = 0; uint32_t size = 0;
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if(taosAscii2Hex(row[i], charLen, &data, &size) < 0){ if (taosAscii2Hex(row[i], charLen, &data, &size) < 0) {
break; break;
} }
(void)memcpy(str + len, data, size); (void)memcpy(str + len, data, size);
len += size; len += size;
taosMemoryFree(data); taosMemoryFree(data);
}break; } break;
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: { case TSDB_DATA_TYPE_GEOMETRY: {
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) { if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY ||
fields[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (charLen > fields[i].bytes || charLen < 0) { if (charLen > fields[i].bytes || charLen < 0) {
tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes);
break; break;
@ -664,7 +665,8 @@ const char *taos_get_client_info() { return version; }
// return int32_t // return int32_t
int taos_affected_rows(TAOS_RES *res) { int taos_affected_rows(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return 0; return 0;
} }
@ -675,7 +677,8 @@ int taos_affected_rows(TAOS_RES *res) {
// return int64_t // return int64_t
int64_t taos_affected_rows64(TAOS_RES *res) { int64_t taos_affected_rows64(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return 0; return 0;
} }
@ -725,7 +728,8 @@ int taos_select_db(TAOS *taos, const char *db) {
} }
void taos_stop_query(TAOS_RES *res) { void taos_stop_query(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return; return;
} }
@ -784,7 +788,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
return pRequest->code; return pRequest->code;
} else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SReqResultInfo *pResultInfo = NULL; SReqResultInfo *pResultInfo = NULL;
int32_t code = tmqGetNextResInfo(res, true, &pResultInfo); int32_t code = tmqGetNextResInfo(res, true, &pResultInfo);
if (code != 0) return code; if (code != 0) return code;
pResultInfo->current = pResultInfo->numOfRows; pResultInfo->current = pResultInfo->numOfRows;
@ -807,7 +811,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SReqResultInfo *pResultInfo = NULL; SReqResultInfo *pResultInfo = NULL;
int32_t code = tmqGetNextResInfo(res, false, &pResultInfo); int32_t code = tmqGetNextResInfo(res, false, &pResultInfo);
if (code != 0) { if (code != 0) {
(*numOfRows) = 0; (*numOfRows) = 0;
return 0; return 0;
@ -937,7 +941,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
SRequestObj *pRequest = pWrapper->pRequest; SRequestObj *pRequest = pWrapper->pRequest;
SQuery *pQuery = pRequest->pQuery; SQuery *pQuery = pRequest->pQuery;
qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); qDebug("0x%" PRIx64 " start to semantic analysis, QID:0x%" PRIx64, pRequest->self, pRequest->requestId);
int64_t analyseStart = taosGetTimestampUs(); int64_t analyseStart = taosGetTimestampUs();
pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart; pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart;
@ -953,7 +957,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
(void)memcpy(&pRequest->parseMeta, pResultMeta, sizeof(*pResultMeta)); (void)memcpy(&pRequest->parseMeta, pResultMeta, sizeof(*pResultMeta));
(void)memset(pResultMeta, 0, sizeof(*pResultMeta)); (void)memset(pResultMeta, 0, sizeof(*pResultMeta));
} }
handleQueryAnslyseRes(pWrapper, pResultMeta, code); handleQueryAnslyseRes(pWrapper, pResultMeta, code);
} }
@ -999,7 +1003,7 @@ void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResult
} }
pNewRequest->pQuery = NULL; pNewRequest->pQuery = NULL;
code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pNewRequest->pQuery); code = nodesMakeNode(QUERY_NODE_QUERY, (SNode **)&pNewRequest->pQuery);
if (pNewRequest->pQuery) { if (pNewRequest->pQuery) {
pNewRequest->pQuery->pRoot = pRoot; pNewRequest->pQuery->pRoot = pRoot;
pRoot = NULL; pRoot = NULL;
@ -1056,15 +1060,15 @@ void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta
pRequest->pQuery = NULL; pRequest->pQuery = NULL;
if (NEED_CLIENT_HANDLE_ERROR(code)) { if (NEED_CLIENT_HANDLE_ERROR(code)) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64,
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId);
restartAsyncQuery(pRequest, code); restartAsyncQuery(pRequest, code);
return; return;
} }
// return to app directly // return to app directly
tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:0x%" PRIx64, pRequest->self, tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, QID:0x%" PRIx64, pRequest->self, tstrerror(code),
tstrerror(code), pRequest->requestId); pRequest->requestId);
pRequest->code = code; pRequest->code = code;
returnToUser(pRequest); returnToUser(pRequest);
} }
@ -1113,7 +1117,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
SQuery *pQuery = pRequest->pQuery; SQuery *pQuery = pRequest->pQuery;
pRequest->metric.ctgCostUs += taosGetTimestampUs() - pRequest->metric.ctgStart; pRequest->metric.ctgCostUs += taosGetTimestampUs() - pRequest->metric.ctgStart;
qDebug("0x%" PRIx64 " start to continue parse, reqId:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId, qDebug("0x%" PRIx64 " start to continue parse, QID:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId,
tstrerror(code)); tstrerror(code));
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
@ -1126,7 +1130,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
} }
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code,
tstrerror(code), pWrapper->pRequest->requestId); tstrerror(code), pWrapper->pRequest->requestId);
destorySqlCallbackWrapper(pWrapper); destorySqlCallbackWrapper(pWrapper);
pRequest->pWrapper = NULL; pRequest->pWrapper = NULL;
@ -1143,7 +1147,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
} }
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code,
tstrerror(code), pWrapper->pRequest->requestId); tstrerror(code), pWrapper->pRequest->requestId);
destorySqlCallbackWrapper(pWrapper); destorySqlCallbackWrapper(pWrapper);
pRequest->pWrapper = NULL; pRequest->pWrapper = NULL;
@ -1261,7 +1265,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
} }
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code, tstrerror(code),
pRequest->requestId); pRequest->requestId);
destorySqlCallbackWrapper(pWrapper); destorySqlCallbackWrapper(pWrapper);
pRequest->pWrapper = NULL; pRequest->pWrapper = NULL;
@ -1269,9 +1273,9 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
pRequest->pQuery = NULL; pRequest->pQuery = NULL;
if (NEED_CLIENT_HANDLE_ERROR(code)) { if (NEED_CLIENT_HANDLE_ERROR(code)) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64,
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId);
(void)refreshMeta(pRequest->pTscObj, pRequest); //ignore return code,try again (void)refreshMeta(pRequest->pTscObj, pRequest); // ignore return code,try again
pRequest->prevCode = code; pRequest->prevCode = code;
doAsyncQuery(pRequest, true); doAsyncQuery(pRequest, true);
return; return;
@ -1285,7 +1289,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest); tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest);
SRequestObj* pUserReq = pRequest; SRequestObj *pUserReq = pRequest;
(void)acquireRequest(pRequest->self); (void)acquireRequest(pRequest->self);
while (pUserReq) { while (pUserReq) {
if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) { if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) {
@ -1631,7 +1635,6 @@ TAOS_STMT *taos_stmt_init_with_options(TAOS *taos, TAOS_STMT_OPTIONS *options) {
return pStmt; return pStmt;
} }
int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
if (stmt == NULL || sql == NULL) { if (stmt == NULL || sql == NULL) {
tscError("NULL parameter for %s", __FUNCTION__); tscError("NULL parameter for %s", __FUNCTION__);
@ -1874,7 +1877,7 @@ int taos_stmt_close(TAOS_STMT *stmt) {
return stmtClose(stmt); return stmtClose(stmt);
} }
int taos_set_conn_mode(TAOS* taos, int mode, int value) { int taos_set_conn_mode(TAOS *taos, int mode, int value) {
if (taos == NULL) { if (taos == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
@ -1897,6 +1900,4 @@ int taos_set_conn_mode(TAOS* taos, int mode, int value) {
return 0; return 0;
} }
char* getBuildInfo(){ char *getBuildInfo() { return buildinfo; }
return buildinfo;
}

141
source/client/src/clientRawBlockWrite.c
View File

@ -23,32 +23,31 @@
#include "tglobal.h" #include "tglobal.h"
#include "tmsgtype.h" #include "tmsgtype.h"
#define RAW_NULL_CHECK(c) \ #define RAW_NULL_CHECK(c) \
do { \ do { \
if (c == NULL) { \ if (c == NULL) { \
code = TSDB_CODE_OUT_OF_MEMORY; \ code = TSDB_CODE_OUT_OF_MEMORY; \
goto end; \ goto end; \
} \ } \
} while (0) } while (0)
#define RAW_FALSE_CHECK(c) \ #define RAW_FALSE_CHECK(c) \
do { \ do { \
if (!c) { \ if (!c) { \
code = TSDB_CODE_INVALID_PARA; \ code = TSDB_CODE_INVALID_PARA; \
goto end; \ goto end; \
} \ } \
} while (0) } while (0)
#define RAW_RETURN_CHECK(c) \ #define RAW_RETURN_CHECK(c) \
do { \ do { \
code = c; \ code = c; \
if (code != 0) { \ if (code != 0) { \
goto end; \ goto end; \
} \ } \
} while (0) } while (0)
#define LOG_ID_TAG "connId:0x%" PRIx64 ",QID:0x%" PRIx64
#define LOG_ID_TAG "connId:0x%" PRIx64 ",reqId:0x%" PRIx64
#define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId #define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId
#define TMQ_META_VERSION "1.0" #define TMQ_META_VERSION "1.0"
@ -57,10 +56,10 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen
static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); } static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t,
int8_t t, SColCmprWrapper* pColCmprRow, cJSON** pJson) { SColCmprWrapper* pColCmprRow, cJSON** pJson) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int8_t buildDefaultCompress = 0; int8_t buildDefaultCompress = 0;
if (pColCmprRow->nCols <= 0) { if (pColCmprRow->nCols <= 0) {
buildDefaultCompress = 1; buildDefaultCompress = 1;
} }
@ -82,7 +81,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche
cJSON* columns = cJSON_CreateArray(); cJSON* columns = cJSON_CreateArray();
RAW_NULL_CHECK(columns); RAW_NULL_CHECK(columns);
for (int i = 0; i < schemaRow->nCols; i++) { for (int i = 0; i < schemaRow->nCols; i++) {
cJSON* column = cJSON_CreateObject(); cJSON* column = cJSON_CreateObject();
RAW_NULL_CHECK(column); RAW_NULL_CHECK(column);
SSchema* s = schemaRow->pSchema + i; SSchema* s = schemaRow->pSchema + i;
cJSON* cname = cJSON_CreateString(s->name); cJSON* cname = cJSON_CreateString(s->name);
@ -142,7 +141,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche
cJSON* tags = cJSON_CreateArray(); cJSON* tags = cJSON_CreateArray();
RAW_NULL_CHECK(tags); RAW_NULL_CHECK(tags);
for (int i = 0; schemaTag && i < schemaTag->nCols; i++) { for (int i = 0; schemaTag && i < schemaTag->nCols; i++) {
cJSON* tag = cJSON_CreateObject(); cJSON* tag = cJSON_CreateObject();
RAW_NULL_CHECK(tag); RAW_NULL_CHECK(tag);
SSchema* s = schemaTag->pSchema + i; SSchema* s = schemaTag->pSchema + i;
cJSON* tname = cJSON_CreateString(s->name); cJSON* tname = cJSON_CreateString(s->name);
@ -176,7 +175,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
if (encode != 0) { if (encode != 0) {
const char* encodeStr = columnEncodeStr(encode); const char* encodeStr = columnEncodeStr(encode);
RAW_NULL_CHECK(encodeStr); RAW_NULL_CHECK(encodeStr);
cJSON* encodeJson = cJSON_CreateString(encodeStr); cJSON* encodeJson = cJSON_CreateString(encodeStr);
RAW_NULL_CHECK(encodeJson); RAW_NULL_CHECK(encodeJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "encode", encodeJson)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "encode", encodeJson));
return code; return code;
@ -185,7 +184,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
if (compress != 0) { if (compress != 0) {
const char* compressStr = columnCompressStr(compress); const char* compressStr = columnCompressStr(compress);
RAW_NULL_CHECK(compressStr); RAW_NULL_CHECK(compressStr);
cJSON* compressJson = cJSON_CreateString(compressStr); cJSON* compressJson = cJSON_CreateString(compressStr);
RAW_NULL_CHECK(compressJson); RAW_NULL_CHECK(compressJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "compress", compressJson)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "compress", compressJson));
return code; return code;
@ -194,7 +193,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
if (level != 0) { if (level != 0) {
const char* levelStr = columnLevelStr(level); const char* levelStr = columnLevelStr(level);
RAW_NULL_CHECK(levelStr); RAW_NULL_CHECK(levelStr);
cJSON* levelJson = cJSON_CreateString(levelStr); cJSON* levelJson = cJSON_CreateString(levelStr);
RAW_NULL_CHECK(levelJson); RAW_NULL_CHECK(levelJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "level", levelJson)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "level", levelJson));
return code; return code;
@ -235,7 +234,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_ADD_COLUMN: { case TSDB_ALTER_TABLE_ADD_COLUMN: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0); TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field); RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name); cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName); RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colType = cJSON_CreateNumber(field->type); cJSON* colType = cJSON_CreateNumber(field->type);
@ -259,7 +258,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: { case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: {
SFieldWithOptions* field = taosArrayGet(req.pFields, 0); SFieldWithOptions* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field); RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name); cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName); RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colType = cJSON_CreateNumber(field->type); cJSON* colType = cJSON_CreateNumber(field->type);
@ -285,7 +284,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_DROP_COLUMN: { case TSDB_ALTER_TABLE_DROP_COLUMN: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0); TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field); RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name); cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName); RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
break; break;
@ -294,7 +293,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0); TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field); RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name); cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName); RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colType = cJSON_CreateNumber(field->type); cJSON* colType = cJSON_CreateNumber(field->type);
@ -320,7 +319,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
RAW_NULL_CHECK(oldField); RAW_NULL_CHECK(oldField);
TAOS_FIELD* newField = taosArrayGet(req.pFields, 1); TAOS_FIELD* newField = taosArrayGet(req.pFields, 1);
RAW_NULL_CHECK(newField); RAW_NULL_CHECK(newField);
cJSON* colName = cJSON_CreateString(oldField->name); cJSON* colName = cJSON_CreateString(oldField->name);
RAW_NULL_CHECK(colName); RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colNewName = cJSON_CreateString(newField->name); cJSON* colNewName = cJSON_CreateString(newField->name);
@ -331,7 +330,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: { case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0); TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field); RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name); cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName); RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
RAW_RETURN_CHECK(setCompressOption(json, field->bytes)); RAW_RETURN_CHECK(setCompressOption(json, field->bytes));
@ -405,7 +404,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(tagNumJson); RAW_NULL_CHECK(tagNumJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tagNum", tagNumJson)); RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tagNum", tagNumJson));
cJSON* tags = cJSON_CreateArray(); cJSON* tags = cJSON_CreateArray();
RAW_NULL_CHECK(tags); RAW_NULL_CHECK(tags);
SArray* pTagVals = NULL; SArray* pTagVals = NULL;
RAW_RETURN_CHECK(tTagToValArray(pTag, &pTagVals)); RAW_RETURN_CHECK(tTagToValArray(pTag, &pTagVals));
@ -416,13 +415,13 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
uError("p->nTag == 0"); uError("p->nTag == 0");
goto end; goto end;
} }
char* pJson = NULL; char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson); parseTagDatatoJson(pTag, &pJson);
cJSON* tag = cJSON_CreateObject(); cJSON* tag = cJSON_CreateObject();
RAW_NULL_CHECK(tag); RAW_NULL_CHECK(tag);
STagVal* pTagVal = taosArrayGet(pTagVals, 0); STagVal* pTagVal = taosArrayGet(pTagVals, 0);
RAW_NULL_CHECK(pTagVal); RAW_NULL_CHECK(pTagVal);
char* ptname = taosArrayGet(tagName, 0); char* ptname = taosArrayGet(tagName, 0);
RAW_NULL_CHECK(ptname); RAW_NULL_CHECK(ptname);
cJSON* tname = cJSON_CreateString(ptname); cJSON* tname = cJSON_CreateString(ptname);
RAW_NULL_CHECK(tname); RAW_NULL_CHECK(tname);
@ -443,7 +442,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(pTagVal); RAW_NULL_CHECK(pTagVal);
cJSON* tag = cJSON_CreateObject(); cJSON* tag = cJSON_CreateObject();
RAW_NULL_CHECK(tag); RAW_NULL_CHECK(tag);
char* ptname = taosArrayGet(tagName, i); char* ptname = taosArrayGet(tagName, i);
RAW_NULL_CHECK(ptname); RAW_NULL_CHECK(ptname);
cJSON* tname = cJSON_CreateString(ptname); cJSON* tname = cJSON_CreateString(ptname);
RAW_NULL_CHECK(tname); RAW_NULL_CHECK(tname);
@ -463,7 +462,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(buf); RAW_NULL_CHECK(buf);
if (!buf) goto end; if (!buf) goto end;
if(dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) { if (dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) {
taosMemoryFree(buf); taosMemoryFree(buf);
goto end; goto end;
} }
@ -489,8 +488,8 @@ end:
static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSON** pJson) { static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSON** pJson) {
int32_t code = 0; int32_t code = 0;
char* string = NULL; char* string = NULL;
cJSON* json = cJSON_CreateObject(); cJSON* json = cJSON_CreateObject();
RAW_NULL_CHECK(json); RAW_NULL_CHECK(json);
cJSON* type = cJSON_CreateString("create"); cJSON* type = cJSON_CreateString("create");
RAW_NULL_CHECK(type); RAW_NULL_CHECK(type);
@ -534,8 +533,8 @@ static void processCreateTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
if (pCreateReq->type == TSDB_CHILD_TABLE) { if (pCreateReq->type == TSDB_CHILD_TABLE) {
buildCreateCTableJson(req.pReqs, req.nReqs, pJson); buildCreateCTableJson(req.pReqs, req.nReqs, pJson);
} else if (pCreateReq->type == TSDB_NORMAL_TABLE) { } else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE,
TSDB_NORMAL_TABLE, &pCreateReq->colCmpr, pJson); &pCreateReq->colCmpr, pJson);
} }
} }
@ -548,7 +547,7 @@ end:
static void processAutoCreateTable(STaosxRsp* rsp, char** string) { static void processAutoCreateTable(STaosxRsp* rsp, char** string) {
SDecoder* decoder = NULL; SDecoder* decoder = NULL;
SVCreateTbReq* pCreateReq = NULL; SVCreateTbReq* pCreateReq = NULL;
int32_t code = 0; int32_t code = 0;
uDebug("auto create table data:%p", rsp); uDebug("auto create table data:%p", rsp);
if (rsp->createTableNum <= 0) { if (rsp->createTableNum <= 0) {
uError("processAutoCreateTable rsp->createTableNum <= 0"); uError("processAutoCreateTable rsp->createTableNum <= 0");
@ -563,7 +562,7 @@ static void processAutoCreateTable(STaosxRsp* rsp, char** string) {
// loop to create table // loop to create table
for (int32_t iReq = 0; iReq < rsp->createTableNum; iReq++) { for (int32_t iReq = 0; iReq < rsp->createTableNum; iReq++) {
// decode // decode
void** data = taosArrayGet(rsp->createTableReq, iReq); void** data = taosArrayGet(rsp->createTableReq, iReq);
RAW_NULL_CHECK(data); RAW_NULL_CHECK(data);
int32_t* len = taosArrayGet(rsp->createTableLen, iReq); int32_t* len = taosArrayGet(rsp->createTableLen, iReq);
RAW_NULL_CHECK(len); RAW_NULL_CHECK(len);
@ -735,7 +734,8 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 3, 1); buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 3, 1);
} }
RAW_NULL_CHECK(buf); RAW_NULL_CHECK(buf);
if(dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) != TSDB_CODE_SUCCESS) { if (dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) !=
TSDB_CODE_SUCCESS) {
taosMemoryFree(buf); taosMemoryFree(buf);
goto end; goto end;
} }
@ -823,7 +823,7 @@ static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
// getTbName(req.tableFName); // getTbName(req.tableFName);
char sql[256] = {0}; char sql[256] = {0};
(void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, (void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
req.tsColName, req.skey, req.tsColName, req.ekey); req.tsColName, req.skey, req.tsColName, req.ekey);
json = cJSON_CreateObject(); json = cJSON_CreateObject();
RAW_NULL_CHECK(json); RAW_NULL_CHECK(json);
@ -865,7 +865,7 @@ static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
RAW_NULL_CHECK(tableNameList); RAW_NULL_CHECK(tableNameList);
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
SVDropTbReq* pDropTbReq = req.pReqs + iReq; SVDropTbReq* pDropTbReq = req.pReqs + iReq;
cJSON* tableName = cJSON_CreateString(pDropTbReq->name); cJSON* tableName = cJSON_CreateString(pDropTbReq->name);
RAW_NULL_CHECK(tableName); RAW_NULL_CHECK(tableName);
RAW_FALSE_CHECK(cJSON_AddItemToArray(tableNameList, tableName)); RAW_FALSE_CHECK(cJSON_AddItemToArray(tableNameList, tableName));
} }
@ -910,7 +910,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
} }
// build create stable // build create stable
pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SFieldWithOptions)); pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SFieldWithOptions));
RAW_NULL_CHECK (pReq.pColumns); RAW_NULL_CHECK(pReq.pColumns);
for (int32_t i = 0; i < req.schemaRow.nCols; i++) { for (int32_t i = 0; i < req.schemaRow.nCols; i++) {
SSchema* pSchema = req.schemaRow.pSchema + i; SSchema* pSchema = req.schemaRow.pSchema + i;
SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes}; SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes};
@ -957,7 +957,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
} }
pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen);
RAW_NULL_CHECK(pCmdMsg.pMsg); RAW_NULL_CHECK(pCmdMsg.pMsg);
if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0){ if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) {
code = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
goto end; goto end;
} }
@ -968,7 +968,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.msgType = pQuery.pCmdMsg->msgType;
pQuery.stableQuery = true; pQuery.stableQuery = true;
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); //ignore, because return value is pRequest (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest
if (pRequest->code == TSDB_CODE_SUCCESS) { if (pRequest->code == TSDB_CODE_SUCCESS) {
SCatalog* pCatalog = NULL; SCatalog* pCatalog = NULL;
@ -1021,7 +1021,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
.requestObjRefId = pRequest->self, .requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
SName pName = {0}; SName pName = {0};
(void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); // ignore the return value, always return pName (void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name,
&pName); // ignore the return value, always return pName
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
@ -1059,19 +1060,18 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
} }
pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen);
RAW_NULL_CHECK(pCmdMsg.pMsg); RAW_NULL_CHECK(pCmdMsg.pMsg);
if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0){ if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) {
code = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
goto end; goto end;
} }
SQuery pQuery = {0}; SQuery pQuery = {0};
pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.execMode = QUERY_EXEC_MODE_RPC;
pQuery.pCmdMsg = &pCmdMsg; pQuery.pCmdMsg = &pCmdMsg;
pQuery.msgType = pQuery.pCmdMsg->msgType; pQuery.msgType = pQuery.pCmdMsg->msgType;
pQuery.stableQuery = true; pQuery.stableQuery = true;
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); //ignore, because return value is pRequest (void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest
if (pRequest->code == TSDB_CODE_SUCCESS) { if (pRequest->code == TSDB_CODE_SUCCESS) {
// ignore the error code // ignore the error code
@ -1391,9 +1391,9 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
} }
(void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, (void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
req.tsColName, req.skey, req.tsColName, req.ekey); req.tsColName, req.skey, req.tsColName, req.ekey);
TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX); TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX);
RAW_NULL_CHECK(res); RAW_NULL_CHECK(res);
SRequestObj* pRequest = (SRequestObj*)res; SRequestObj* pRequest = (SRequestObj*)res;
code = pRequest->code; code = pRequest->code;
@ -1500,7 +1500,6 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
if (TSDB_CODE_SUCCESS != code) goto end; if (TSDB_CODE_SUCCESS != code) goto end;
RAW_RETURN_CHECK(rewriteToVnodeModifyOpStmt(pQuery, pArray)); RAW_RETURN_CHECK(rewriteToVnodeModifyOpStmt(pQuery, pArray));
(void)launchQueryImpl(pRequest, pQuery, true, NULL); (void)launchQueryImpl(pRequest, pQuery, true, NULL);
pVgData = NULL; pVgData = NULL;
@ -1573,7 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
RAW_RETURN_CHECK(smlInitHandle(&pQuery)); RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash); RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData))); RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0)); RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
@ -1632,7 +1632,8 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
RAW_RETURN_CHECK(smlInitHandle(&pQuery)); RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash); RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData))); RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0)); RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
@ -1735,7 +1736,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter);
RAW_NULL_CHECK(pSW); RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
RAW_NULL_CHECK(fields); RAW_NULL_CHECK(fields);
for (int i = 0; i < pSW->nCols; i++) { for (int i = 0; i < pSW->nCols; i++) {
fields[i].type = pSW->pSchema[i].type; fields[i].type = pSW->pSchema[i].type;
@ -1844,7 +1845,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
// find schema data info // find schema data info
for (int j = 0; j < rspObj.rsp.createTableNum; j++) { for (int j = 0; j < rspObj.rsp.createTableNum; j++) {
void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
RAW_NULL_CHECK(dataTmp); RAW_NULL_CHECK(dataTmp);
int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j); int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j);
RAW_NULL_CHECK(dataTmp); RAW_NULL_CHECK(dataTmp);
@ -1896,7 +1897,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter);
RAW_NULL_CHECK(pSW); RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
if (fields == NULL) { if (fields == NULL) {
SET_ERROR_MSG("calloc fields failed"); SET_ERROR_MSG("calloc fields failed");
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -1979,10 +1980,10 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) {
RAW_NULL_CHECK(len); RAW_NULL_CHECK(len);
void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i); void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i);
RAW_NULL_CHECK(tmpBuf); RAW_NULL_CHECK(tmpBuf);
SDecoder metaCoder = {0}; SDecoder metaCoder = {0};
SMqMetaRsp metaRsp = {0}; SMqMetaRsp metaRsp = {0};
tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), *len - sizeof(SMqRspHead)); tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), *len - sizeof(SMqRspHead));
if(tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0 ) { if (tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0) {
goto end; goto end;
} }
cJSON* pItem = NULL; cJSON* pItem = NULL;
@ -2012,18 +2013,18 @@ char* tmq_get_json_meta(TAOS_RES* res) {
if (TD_RES_TMQ_METADATA(res)) { if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res; SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res;
char* string = NULL; char* string = NULL;
processAutoCreateTable(&pMetaDataRspObj->rsp, &string); processAutoCreateTable(&pMetaDataRspObj->rsp, &string);
return string; return string;
} else if (TD_RES_TMQ_BATCH_META(res)) { } else if (TD_RES_TMQ_BATCH_META(res)) {
SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res; SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res;
char* string = NULL; char* string = NULL;
processBatchMetaToJson(&pBatchMetaRspObj->rsp, &string); processBatchMetaToJson(&pBatchMetaRspObj->rsp, &string);
return string; return string;
} }
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
cJSON* pJson = NULL; cJSON* pJson = NULL;
processSimpleMeta(&pMetaRspObj->metaRsp, &pJson); processSimpleMeta(&pMetaRspObj->metaRsp, &pJson);
char* string = cJSON_PrintUnformatted(pJson); char* string = cJSON_PrintUnformatted(pJson);
cJSON_Delete(pJson); cJSON_Delete(pJson);
@ -2102,7 +2103,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
uDebug("tmq get raw type meta:%p", raw); uDebug("tmq get raw type meta:%p", raw);
} else if (TD_RES_TMQ(res)) { } else if (TD_RES_TMQ(res)) {
SMqRspObj* rspObj = ((SMqRspObj*)res); SMqRspObj* rspObj = ((SMqRspObj*)res);
int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw); int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw);
if (code != 0) { if (code != 0) {
uError("tmq get raw type error:%d", terrno); uError("tmq get raw type error:%d", terrno);
return code; return code;

235
source/client/src/clientTmq.c
View File

@ -37,7 +37,7 @@ struct SMqMgmt {
static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once
volatile int32_t tmqInitRes = 0; // initialize rsp code volatile int32_t tmqInitRes = 0; // initialize rsp code
static struct SMqMgmt tmqMgmt = {0}; static struct SMqMgmt tmqMgmt = {0};
static int8_t pollFlag = 0; static int8_t pollFlag = 0;
typedef struct { typedef struct {
int32_t code; int32_t code;
@ -121,7 +121,7 @@ struct tmq_t {
typedef struct SAskEpInfo { typedef struct SAskEpInfo {
int32_t code; int32_t code;
tsem2_t sem; tsem2_t sem;
} SAskEpInfo; } SAskEpInfo;
enum { enum {
@ -191,7 +191,7 @@ typedef struct {
} SMqPollRspWrapper; } SMqPollRspWrapper;
typedef struct { typedef struct {
tsem2_t rspSem; tsem2_t rspSem;
int32_t rspErr; int32_t rspErr;
} SMqSubscribeCbParam; } SMqSubscribeCbParam;
@ -219,12 +219,12 @@ typedef struct SMqVgCommon {
} SMqVgCommon; } SMqVgCommon;
typedef struct SMqSeekParam { typedef struct SMqSeekParam {
tsem2_t sem; tsem2_t sem;
int32_t code; int32_t code;
} SMqSeekParam; } SMqSeekParam;
typedef struct SMqCommittedParam { typedef struct SMqCommittedParam {
tsem2_t sem; tsem2_t sem;
int32_t code; int32_t code;
SMqVgOffset vgOffset; SMqVgOffset vgOffset;
} SMqCommittedParam; } SMqCommittedParam;
@ -242,18 +242,18 @@ typedef struct {
int32_t waitingRspNum; int32_t waitingRspNum;
int32_t code; int32_t code;
tmq_commit_cb* callbackFn; tmq_commit_cb* callbackFn;
void* userParam; void* userParam;
} SMqCommitCbParamSet; } SMqCommitCbParamSet;
typedef struct { typedef struct {
SMqCommitCbParamSet* params; SMqCommitCbParamSet* params;
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
int32_t vgId; int32_t vgId;
int64_t consumerId; int64_t consumerId;
} SMqCommitCbParam; } SMqCommitCbParam;
typedef struct SSyncCommitInfo { typedef struct SSyncCommitInfo {
tsem2_t sem; tsem2_t sem;
int32_t code; int32_t code;
} SSyncCommitInfo; } SSyncCommitInfo;
@ -334,7 +334,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcasecmp(key, "session.timeout.ms") == 0) { if (strcasecmp(key, "session.timeout.ms") == 0) {
int64_t tmp = taosStr2int64(value); int64_t tmp = taosStr2int64(value);
if (tmp < 6000 || tmp > 1800000){ if (tmp < 6000 || tmp > 1800000) {
return TMQ_CONF_INVALID; return TMQ_CONF_INVALID;
} }
conf->sessionTimeoutMs = tmp; conf->sessionTimeoutMs = tmp;
@ -343,7 +343,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcasecmp(key, "heartbeat.interval.ms") == 0) { if (strcasecmp(key, "heartbeat.interval.ms") == 0) {
int64_t tmp = taosStr2int64(value); int64_t tmp = taosStr2int64(value);
if (tmp < 1000 || tmp >= conf->sessionTimeoutMs){ if (tmp < 1000 || tmp >= conf->sessionTimeoutMs) {
return TMQ_CONF_INVALID; return TMQ_CONF_INVALID;
} }
conf->heartBeatIntervalMs = tmp; conf->heartBeatIntervalMs = tmp;
@ -352,7 +352,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcasecmp(key, "max.poll.interval.ms") == 0) { if (strcasecmp(key, "max.poll.interval.ms") == 0) {
int64_t tmp = taosStr2int64(value); int64_t tmp = taosStr2int64(value);
if (tmp < 1000 || tmp > INT32_MAX){ if (tmp < 1000 || tmp > INT32_MAX) {
return TMQ_CONF_INVALID; return TMQ_CONF_INVALID;
} }
conf->maxPollIntervalMs = tmp; conf->maxPollIntervalMs = tmp;
@ -515,7 +515,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len); tEncoderInit(&encoder, abuf, len);
if(tEncodeMqVgOffset(&encoder, &pOffset) < 0) { if (tEncodeMqVgOffset(&encoder, &pOffset) < 0) {
tEncoderClear(&encoder); tEncoderClear(&encoder);
taosMemoryFree(buf); taosMemoryFree(buf);
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
@ -562,7 +562,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse
return code; return code;
} }
static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic **topic) { static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic** topic) {
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
for (int32_t i = 0; i < numOfTopics; ++i) { for (int32_t i = 0; i < numOfTopics; ++i) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
@ -577,8 +577,8 @@ static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic
return TSDB_CODE_TMQ_INVALID_TOPIC; return TSDB_CODE_TMQ_INVALID_TOPIC;
} }
static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, int32_t rspNum,
int32_t rspNum, SMqCommitCbParamSet** ppParamSet) { SMqCommitCbParamSet** ppParamSet) {
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
if (pParamSet == NULL) { if (pParamSet == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -595,7 +595,7 @@ static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, voi
static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg) { static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg) {
SMqClientTopic* pTopic = NULL; SMqClientTopic* pTopic = NULL;
int32_t code = getTopicByName(tmq, pTopicName, &pTopic); int32_t code = getTopicByName(tmq, pTopicName, &pTopic);
if (code != 0) { if (code != 0) {
tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName);
return code; return code;
@ -723,7 +723,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
taosRUnLockLatch(&tmq->lock); taosRUnLockLatch(&tmq->lock);
goto end; goto end;
} }
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName,
numOfVgroups); numOfVgroups);
for (int32_t j = 0; j < numOfVgroups; j++) { for (int32_t j = 0; j < numOfVgroups; j++) {
@ -769,7 +769,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
if (pParamSet->waitingRspNum != 1) { if (pParamSet->waitingRspNum != 1) {
// count down since waiting rsp num init as 1 // count down since waiting rsp num init as 1
code = commitRspCountDown(pParamSet, tmq->consumerId, "", 0); code = commitRspCountDown(pParamSet, tmq->consumerId, "", 0);
if (code != 0){ if (code != 0) {
tscError("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code)); tscError("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code));
pParamSet = NULL; pParamSet = NULL;
goto end; goto end;
@ -824,14 +824,14 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
} }
int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
if (code != 0){ if (code != 0) {
goto _return; goto _return;
} }
if (pMsg == NULL || param == NULL) { if (pMsg == NULL || param == NULL) {
code = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
goto _return; goto _return;
} }
SMqHbRsp rsp = {0}; SMqHbRsp rsp = {0};
code = tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp); code = tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp);
if (code != 0) { if (code != 0) {
@ -858,7 +858,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
taosWUnLockLatch(&tmq->lock); taosWUnLockLatch(&tmq->lock);
(void)taosReleaseRef(tmqMgmt.rsetId, refId); (void)taosReleaseRef(tmqMgmt.rsetId, refId);
} }
tDestroySMqHbRsp(&rsp); tDestroySMqHbRsp(&rsp);
_return: _return:
@ -881,32 +881,32 @@ void tmqSendHbReq(void* param, void* tmrId) {
req.epoch = tmq->epoch; req.epoch = tmq->epoch;
req.pollFlag = atomic_load_8(&pollFlag); req.pollFlag = atomic_load_8(&pollFlag);
req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
if (req.topics == NULL){ if (req.topics == NULL) {
return; return;
} }
taosRLockLatch(&tmq->lock); taosRLockLatch(&tmq->lock);
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (pTopic == NULL) { if (pTopic == NULL) {
continue; continue;
} }
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
TopicOffsetRows* data = taosArrayReserve(req.topics, 1); TopicOffsetRows* data = taosArrayReserve(req.topics, 1);
if (data == NULL){ if (data == NULL) {
continue; continue;
} }
(void)strcpy(data->topicName, pTopic->topicName); (void)strcpy(data->topicName, pTopic->topicName);
data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows));
if (data->offsetRows == NULL){ if (data->offsetRows == NULL) {
continue; continue;
} }
for (int j = 0; j < numOfVgroups; j++) { for (int j = 0; j < numOfVgroups; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
if (pVg == NULL){ if (pVg == NULL) {
continue; continue;
} }
OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1);
if (offRows == NULL){ if (offRows == NULL) {
continue; continue;
} }
offRows->vgId = pVg->vgId; offRows->vgId = pVg->vgId;
@ -964,7 +964,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
(void)atomic_val_compare_exchange_8(&pollFlag, 1, 0); (void)atomic_val_compare_exchange_8(&pollFlag, 1, 0);
OVER: OVER:
tDestroySMqHbReq(&req); tDestroySMqHbReq(&req);
if(tmrId != NULL){ if (tmrId != NULL) {
(void)taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); (void)taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer);
} }
(void)taosReleaseRef(tmqMgmt.rsetId, refId); (void)taosReleaseRef(tmqMgmt.rsetId, refId);
@ -1006,14 +1006,15 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
continue; continue;
} }
tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
(void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer); (void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->epTimer);
} else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn; tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn;
asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId,
pTmq->autoCommitInterval / 1000.0); pTmq->autoCommitInterval / 1000.0);
(void)taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, (void)taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->commitTimer); &pTmq->commitTimer);
} else { } else {
tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
} }
@ -1100,16 +1101,16 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
taosRLockLatch(&tmq->lock); taosRLockLatch(&tmq->lock);
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i);
if(topic == NULL) { if (topic == NULL) {
tscError("topic is null"); tscError("topic is null");
continue; continue;
} }
char* tmp = strchr(topic->topicName, '.'); char* tmp = strchr(topic->topicName, '.');
if(tmp == NULL) { if (tmp == NULL) {
tscError("topic name is invalid:%s", topic->topicName); tscError("topic name is invalid:%s", topic->topicName);
continue; continue;
} }
if(tmq_list_append(*topics, tmp+ 1) != 0) { if (tmq_list_append(*topics, tmp + 1) != 0) {
tscError("failed to append topic:%s", tmp + 1); tscError("failed to append topic:%s", tmp + 1);
continue; continue;
} }
@ -1227,27 +1228,31 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
} }
code = taosOpenQueue(&pTmq->mqueue); code = taosOpenQueue(&pTmq->mqueue);
if (code) { if (code) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("open queue failed") SET_ERROR_MSG_TMQ("open queue failed")
goto _failed; goto _failed;
} }
code = taosOpenQueue(&pTmq->delayedTask); code = taosOpenQueue(&pTmq->delayedTask);
if (code) { if (code) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("open delayed task queue failed") SET_ERROR_MSG_TMQ("open delayed task queue failed")
goto _failed; goto _failed;
} }
code = taosAllocateQall(&pTmq->qall); code = taosAllocateQall(&pTmq->qall);
if (code) { if (code) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("allocate qall failed") SET_ERROR_MSG_TMQ("allocate qall failed")
goto _failed; goto _failed;
} }
if (conf->groupId[0] == 0) { if (conf->groupId[0] == 0) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId); tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty") SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty")
goto _failed; goto _failed;
} }
@ -1287,8 +1292,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
// init semaphore // init semaphore
if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) { if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) {
tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, tstrerror(TAOS_SYSTEM_ERROR(errno)), tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId,
pTmq->groupId); tstrerror(TAOS_SYSTEM_ERROR(errno)), pTmq->groupId);
SET_ERROR_MSG_TMQ("init t_sem failed") SET_ERROR_MSG_TMQ("init t_sem failed")
goto _failed; goto _failed;
} }
@ -1371,7 +1376,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
SName name = {0}; SName name = {0};
code = tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic)); code = tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic));
if (code) { if (code) {
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, code); tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId,
code);
goto FAIL; goto FAIL;
} }
char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
@ -1382,7 +1388,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
code = tNameExtractFullName(&name, topicFName); code = tNameExtractFullName(&name, topicFName);
if (code) { if (code) {
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, code); tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId,
code);
taosMemoryFree(topicFName); taosMemoryFree(topicFName);
goto FAIL; goto FAIL;
} }
@ -1459,7 +1466,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
} }
tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer); tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer);
tmq->commitTimer =taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); tmq->commitTimer =
taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer);
if (tmq->epTimer == NULL || tmq->commitTimer == NULL) { if (tmq->epTimer == NULL || tmq->commitTimer == NULL) {
code = TSDB_CODE_TSC_INTERNAL_ERROR; code = TSDB_CODE_TSC_INTERNAL_ERROR;
goto FAIL; goto FAIL;
@ -1516,12 +1524,12 @@ static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId) {
} }
int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tmq_t* tmq = NULL; tmq_t* tmq = NULL;
SMqPollRspWrapper* pRspWrapper = NULL; SMqPollRspWrapper* pRspWrapper = NULL;
int8_t rspType = 0; int8_t rspType = 0;
int32_t vgId = 0; int32_t vgId = 0;
uint64_t requestId = 0; uint64_t requestId = 0;
SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqPollCbParam* pParam = (SMqPollCbParam*)param;
if (pMsg == NULL) { if (pMsg == NULL) {
return TSDB_CODE_TSC_INTERNAL_ERROR; return TSDB_CODE_TSC_INTERNAL_ERROR;
} }
@ -1530,7 +1538,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFreeClear(pMsg->pEpSet); taosMemoryFreeClear(pMsg->pEpSet);
return TSDB_CODE_TSC_INTERNAL_ERROR; return TSDB_CODE_TSC_INTERNAL_ERROR;
} }
int64_t refId = pParam->refId; int64_t refId = pParam->refId;
vgId = pParam->vgId; vgId = pParam->vgId;
requestId = pParam->requestId; requestId = pParam->requestId;
tmq = taosAcquireRef(tmqMgmt.rsetId, refId); tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
@ -1562,7 +1570,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (msgEpoch < clientEpoch) { if (msgEpoch < clientEpoch) {
// do not write into queue since updating epoch reset // do not write into queue since updating epoch reset
tscWarn("consumer:0x%" PRIx64 tscWarn("consumer:0x%" PRIx64
" msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, QID:0x%" PRIx64,
tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId);
code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; code = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
goto END; goto END;
@ -1594,7 +1602,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
char buf[TSDB_OFFSET_LEN] = {0}; char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset);
tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, QID:0x%" PRIx64,
tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId); tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId);
} else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SDecoder decoder = {0}; SDecoder decoder = {0};
@ -1626,23 +1634,24 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
(void)memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead)); (void)memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead));
tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, vgId,requestId); tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, QID:0x%" PRIx64, tmq->consumerId, vgId,
requestId);
} else { // invalid rspType } else { // invalid rspType
tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
} }
END: END:
if (pRspWrapper){ if (pRspWrapper) {
pRspWrapper->code = code; pRspWrapper->code = code;
pRspWrapper->vgId = vgId; pRspWrapper->vgId = vgId;
(void)strcpy(pRspWrapper->topicName, pParam->topicName); (void)strcpy(pRspWrapper->topicName, pParam->topicName);
code = taosWriteQitem(tmq->mqueue, pRspWrapper); code = taosWriteQitem(tmq->mqueue, pRspWrapper);
if(code != 0){ if (code != 0) {
tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code);
} }
} }
int32_t total = taosQueueItemSize(tmq->mqueue); int32_t total = taosQueueItemSize(tmq->mqueue);
tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64, tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, QID:0x%" PRIx64,
tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId); tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId);
if (tmq) (void)tsem2_post(&tmq->rspSem); if (tmq) (void)tsem2_post(&tmq->rspSem);
@ -1681,7 +1690,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
} }
for (int32_t j = 0; j < vgNumGet; j++) { for (int32_t j = 0; j < vgNumGet; j++) {
SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
if (pVgEp == NULL){ if (pVgEp == NULL) {
continue; continue;
} }
(void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId); (void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId);
@ -1717,7 +1726,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
clientVg.offsetInfo.committedOffset = offsetNew; clientVg.offsetInfo.committedOffset = offsetNew;
clientVg.offsetInfo.beginOffset = offsetNew; clientVg.offsetInfo.beginOffset = offsetNew;
} }
if (taosArrayPush(pTopic->vgs, &clientVg) == NULL){ if (taosArrayPush(pTopic->vgs, &clientVg) == NULL) {
tscError("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId, tscError("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId,
pTopic->topicName); pTopic->topicName);
freeClientVg(&clientVg); freeClientVg(&clientVg);
@ -1778,7 +1787,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
.commitOffset = pVgCur->offsetInfo.committedOffset, .commitOffset = pVgCur->offsetInfo.committedOffset,
.numOfRows = pVgCur->numOfRows, .numOfRows = pVgCur->numOfRows,
.vgStatus = pVgCur->vgStatus}; .vgStatus = pVgCur->vgStatus};
if(taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0){ if (taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0) {
tscError("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId); tscError("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId);
} }
} }
@ -1792,7 +1801,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
continue; continue;
} }
initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq); initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq);
if(taosArrayPush(newTopics, &topic) == NULL){ if (taosArrayPush(newTopics, &topic) == NULL) {
tscError("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName); tscError("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName);
freeClientTopic(&topic); freeClientTopic(&topic);
} }
@ -1927,7 +1936,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
pDataRsp->withSchema = true; pDataRsp->withSchema = true;
pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*)); pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*));
if (pDataRsp->blockSchema == NULL){ if (pDataRsp->blockSchema == NULL) {
tscError("failed to allocate memory for blockSchema"); tscError("failed to allocate memory for blockSchema");
return; return;
} }
@ -1946,7 +1955,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
if (schema) { if (schema) {
if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL){ if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL) {
tscError("failed to push schema into blockSchema"); tscError("failed to push schema into blockSchema");
continue; continue;
} }
@ -1955,7 +1964,8 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
} }
} }
int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj** ppRspObj) { int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows,
SMqRspObj** ppRspObj) {
SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
if (pRspObj == NULL) { if (pRspObj == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -1967,7 +1977,8 @@ int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, in
return 0; return 0;
} }
int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqTaosxRspObj** ppRspObj) { int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows,
SMqTaosxRspObj** ppRspObj) {
SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
if (pRspObj == NULL) { if (pRspObj == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -2038,7 +2049,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset);
code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, reqId:0x%" PRIx64, tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, QID:0x%" PRIx64,
pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId); pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId);
if (code != 0) { if (code != 0) {
return code; return code;
@ -2064,10 +2075,10 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
for (int i = 0; i < numOfTopics; i++) { for (int i = 0; i < numOfTopics; i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (pTopic == NULL){ if (pTopic == NULL) {
continue; continue;
} }
int32_t numOfVg = taosArrayGetSize(pTopic->vgs); int32_t numOfVg = taosArrayGetSize(pTopic->vgs);
if (pTopic->noPrivilege) { if (pTopic->noPrivilege) {
tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName); tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName);
continue; continue;
@ -2077,7 +2088,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
if (pVg == NULL) { if (pVg == NULL) {
continue; continue;
} }
int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs;
if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms
tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId,
tmq->epoch, pVg->vgId); tmq->epoch, pVg->vgId);
@ -2211,7 +2222,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset);
if (pDataRsp->blockNum == 0) { if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64
", total:%" PRId64 ", reqId:0x%" PRIx64, ", total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId);
pVg->emptyBlockReceiveTs = taosGetTimestampMs(); pVg->emptyBlockReceiveTs = taosGetTimestampMs();
tmqFreeRspWrapper(pRspWrapper); tmqFreeRspWrapper(pRspWrapper);
@ -2228,13 +2239,13 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
pVg->blockSleepForReplay = pRsp->rsp.sleepTime; pVg->blockSleepForReplay = pRsp->rsp.sleepTime;
if (pVg->blockSleepForReplay > 0) { if (pVg->blockSleepForReplay > 0) {
if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) { if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) {
tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%"PRId64, tmq->consumerId, tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%" PRId64,
pVg->vgId, pVg->blockSleepForReplay); tmq->consumerId, pVg->vgId, pVg->blockSleepForReplay);
} }
} }
} }
tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, ", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId); pollRspWrapper->reqId);
taosFreeQitem(pRspWrapper); taosFreeQitem(pRspWrapper);
@ -2310,7 +2321,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever, pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever,
tmq->consumerId, true); tmq->consumerId, true);
SMqBatchMetaRspObj* pRsp = NULL; SMqBatchMetaRspObj* pRsp = NULL;
(void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp) ; (void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp);
taosFreeQitem(pRspWrapper); taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock); taosWUnLockLatch(&tmq->lock);
return pRsp; return pRsp;
@ -2345,7 +2356,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tmq->consumerId, pDataRsp->blockNum != 0); tmq->consumerId, pDataRsp->blockNum != 0);
if (pDataRsp->blockNum == 0) { if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId); tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId);
pVg->emptyBlockReceiveTs = taosGetTimestampMs(); pVg->emptyBlockReceiveTs = taosGetTimestampMs();
tmqFreeRspWrapper(pRspWrapper); tmqFreeRspWrapper(pRspWrapper);
@ -2357,9 +2368,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
} }
// build rsp // build rsp
int64_t numOfRows = 0; int64_t numOfRows = 0;
SMqTaosxRspObj* pRsp = NULL; SMqTaosxRspObj* pRsp = NULL;
if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) !=0 ) { if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) != 0) {
tscError("consumer:0x%" PRIx64 " build taosx rsp failed, vgId:%d", tmq->consumerId, pVg->vgId); tscError("consumer:0x%" PRIx64 " build taosx rsp failed, vgId:%d", tmq->consumerId, pVg->vgId);
} }
tmq->totalRows += numOfRows; tmq->totalRows += numOfRows;
@ -2367,7 +2378,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
char buf[TSDB_OFFSET_LEN] = {0}; char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, ", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId); pollRspWrapper->reqId);
@ -2479,7 +2490,7 @@ static void displayConsumeStatistics(tmq_t* pTmq) {
tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId);
} }
static int32_t innerClose(tmq_t* tmq){ static int32_t innerClose(tmq_t* tmq) {
if (tmq->status != TMQ_CONSUMER_STATUS__READY) { if (tmq->status != TMQ_CONSUMER_STATUS__READY) {
tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId); tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId);
return 0; return 0;
@ -2493,7 +2504,7 @@ static int32_t innerClose(tmq_t* tmq){
tmqSendHbReq((void*)(tmq->refId), NULL); tmqSendHbReq((void*)(tmq->refId), NULL);
tmq_list_t* lst = tmq_list_new(); tmq_list_t* lst = tmq_list_new();
if (lst == NULL){ if (lst == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
int32_t code = tmq_subscribe(tmq, lst); int32_t code = tmq_subscribe(tmq, lst);
@ -2507,7 +2518,7 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
int32_t code = 0; int32_t code = 0;
if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) {
code = innerClose(tmq); code = innerClose(tmq);
if(code == 0){ if (code == 0) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED);
} }
} }
@ -2522,12 +2533,12 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
int32_t code = 0; int32_t code = 0;
if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) { if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) {
code = innerClose(tmq); code = innerClose(tmq);
if(code == 0){ if (code == 0) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED);
} }
} }
if (code == 0){ if (code == 0) {
(void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId); (void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId);
} }
return code; return code;
@ -2570,13 +2581,13 @@ const char* tmq_get_topic_name(TAOS_RES* res) {
return NULL; return NULL;
} }
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
char *tmp = strchr(((SMqRspObjCommon*)res)->topic, '.'); char* tmp = strchr(((SMqRspObjCommon*)res)->topic, '.');
if (tmp == NULL) { if (tmp == NULL) {
return NULL; return NULL;
} }
return tmp + 1; return tmp + 1;
} else if (TD_RES_TMQ_META(res)) { } else if (TD_RES_TMQ_META(res)) {
char *tmp = strchr(((SMqMetaRspObj*)res)->topic, '.'); char* tmp = strchr(((SMqMetaRspObj*)res)->topic, '.');
if (tmp == NULL) { if (tmp == NULL) {
return NULL; return NULL;
} }
@ -2592,13 +2603,13 @@ const char* tmq_get_db_name(TAOS_RES* res) {
} }
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
char *tmp = strchr(((SMqRspObjCommon*)res)->db, '.'); char* tmp = strchr(((SMqRspObjCommon*)res)->db, '.');
if (tmp == NULL) { if (tmp == NULL) {
return NULL; return NULL;
} }
return tmp + 1; return tmp + 1;
} else if (TD_RES_TMQ_META(res)) { } else if (TD_RES_TMQ_META(res)) {
char *tmp = strchr(((SMqMetaRspObj*)res)->db, '.'); char* tmp = strchr(((SMqMetaRspObj*)res)->db, '.');
if (tmp == NULL) { if (tmp == NULL) {
return NULL; return NULL;
} }
@ -2698,7 +2709,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) {
int32_t code = 0; int32_t code = 0;
SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo));
if(pInfo == NULL) { if (pInfo == NULL) {
tscError("failed to allocate memory for sync commit"); tscError("failed to allocate memory for sync commit");
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -2844,7 +2855,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
if (param == NULL) { if (param == NULL) {
goto FAIL; goto FAIL;
} }
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
if (tmq == NULL) { if (tmq == NULL) {
@ -2865,7 +2876,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch);
if (pParam->sync) { if (pParam->sync) {
SMqAskEpRsp rsp = {0}; SMqAskEpRsp rsp = {0};
if(tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL){ if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL) {
(void)doUpdateLocalEp(tmq, head->epoch, &rsp); (void)doUpdateLocalEp(tmq, head->epoch, &rsp);
} }
tDeleteSMqAskEpRsp(&rsp); tDeleteSMqAskEpRsp(&rsp);
@ -2879,10 +2890,10 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
pWrapper->epoch = head->epoch; pWrapper->epoch = head->epoch;
(void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); (void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL){ if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL) {
tmqFreeRspWrapper((SMqRspWrapper*)pWrapper); tmqFreeRspWrapper((SMqRspWrapper*)pWrapper);
taosFreeQitem(pWrapper); taosFreeQitem(pWrapper);
}else{ } else {
(void)taosWriteQitem(tmq->mqueue, pWrapper); (void)taosWriteQitem(tmq->mqueue, pWrapper);
} }
} }
@ -2980,7 +2991,7 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) {
sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; sendInfo->msgType = TDMT_MND_TMQ_ASK_EP;
SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp);
tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, QID:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId);
return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo);
} }
@ -3023,7 +3034,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
if (common->withSchema) { if (common->withSchema) {
doFreeReqResultInfo(&pRspObj->resInfo); doFreeReqResultInfo(&pRspObj->resInfo);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter); SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter);
if (pSW){ if (pSW) {
TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols)); TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols));
} }
} }
@ -3040,9 +3051,9 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
pRspObj->resInfo.precision = precision; pRspObj->resInfo.precision = precision;
pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows; pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows;
int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, pRspObj->resInfo.numOfRows, int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols,
convertUcs4); pRspObj->resInfo.numOfRows, convertUcs4);
if (code != 0){ if (code != 0) {
return code; return code;
} }
*pResInfo = &pRspObj->resInfo; *pResInfo = &pRspObj->resInfo;
@ -3070,18 +3081,18 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
code = tDecodeMqDataRsp(&decoder, &rsp); code = tDecodeMqDataRsp(&decoder, &rsp);
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (code != 0){ if (code != 0) {
goto END; goto END;
} }
SMqRspHead* pHead = pMsg->pData; SMqRspHead* pHead = pMsg->pData;
tmq_topic_assignment assignment = {.begin = pHead->walsver, tmq_topic_assignment assignment = {.begin = pHead->walsver,
.end = pHead->walever + 1, .end = pHead->walever + 1,
.currentOffset = rsp.common.rspOffset.version, .currentOffset = rsp.common.rspOffset.version,
.vgId = pParam->vgId}; .vgId = pParam->vgId};
(void)taosThreadMutexLock(&pCommon->mutex); (void)taosThreadMutexLock(&pCommon->mutex);
if(taosArrayPush(pCommon->pList, &assignment) == NULL){ if (taosArrayPush(pCommon->pList, &assignment) == NULL) {
tscError("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId, tscError("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId,
pParam->vgId, pCommon->pTopicName); pParam->vgId, pCommon->pTopicName);
code = TSDB_CODE_TSC_INTERNAL_ERROR; code = TSDB_CODE_TSC_INTERNAL_ERROR;
@ -3192,7 +3203,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep
taosMemoryFree(sendInfo); taosMemoryFree(sendInfo);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (tsem2_init(&pParam->sem, 0, 0) != 0){ if (tsem2_init(&pParam->sem, 0, 0) != 0) {
taosMemoryFree(buf); taosMemoryFree(buf);
taosMemoryFree(sendInfo); taosMemoryFree(sendInfo);
taosMemoryFree(pParam); taosMemoryFree(pParam);
@ -3356,7 +3367,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
taosWLockLatch(&tmq->lock); taosWLockLatch(&tmq->lock);
SMqClientTopic* pTopic = NULL; SMqClientTopic* pTopic = NULL;
int32_t code = getTopicByName(tmq, tname, &pTopic); int32_t code = getTopicByName(tmq, tname, &pTopic);
if (code != 0) { if (code != 0) {
tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName);
goto end; goto end;
@ -3366,10 +3377,10 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
*numOfAssignment = taosArrayGetSize(pTopic->vgs); *numOfAssignment = taosArrayGetSize(pTopic->vgs);
for (int32_t j = 0; j < (*numOfAssignment); ++j) { for (int32_t j = 0; j < (*numOfAssignment); ++j) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j);
if (pClientVg == NULL){ if (pClientVg == NULL) {
continue; continue;
} }
int32_t type = pClientVg->offsetInfo.beginOffset.type; int32_t type = pClientVg->offsetInfo.beginOffset.type;
if (isInSnapshotMode(type, tmq->useSnapshot)) { if (isInSnapshotMode(type, tmq->useSnapshot)) {
tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type);
code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; code = TSDB_CODE_TMQ_SNAPSHOT_ERROR;
@ -3389,7 +3400,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
for (int32_t j = 0; j < (*numOfAssignment); ++j) { for (int32_t j = 0; j < (*numOfAssignment); ++j) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j);
if (pClientVg == NULL){ if (pClientVg == NULL) {
continue; continue;
} }
if (pClientVg->offsetInfo.beginOffset.type != TMQ_OFFSET__LOG) { if (pClientVg->offsetInfo.beginOffset.type != TMQ_OFFSET__LOG) {
@ -3418,7 +3429,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
if (tsem2_init(&pCommon->rsp, 0, 0) != 0){ if (tsem2_init(&pCommon->rsp, 0, 0) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
@ -3428,7 +3439,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
for (int32_t i = 0; i < (*numOfAssignment); ++i) { for (int32_t i = 0; i < (*numOfAssignment); ++i) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i);
if (pClientVg == NULL){ if (pClientVg == NULL) {
continue; continue;
} }
SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam)); SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam));
@ -3487,8 +3498,8 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset); tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset);
tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, QID:0x%" PRIx64, tmq->consumerId,
tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo);
if (code != 0) { if (code != 0) {
goto end; goto end;
@ -3512,7 +3523,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
for (int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i);
if (pClientVg == NULL){ if (pClientVg == NULL) {
continue; continue;
} }
if (pClientVg->vgId != p->vgId) { if (pClientVg->vgId != p->vgId) {
@ -3639,7 +3650,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
taosMemoryFree(sendInfo); taosMemoryFree(sendInfo);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (tsem2_init(&pParam->sem, 0, 0) != 0){ if (tsem2_init(&pParam->sem, 0, 0) != 0) {
taosMemoryFree(msg); taosMemoryFree(msg);
taosMemoryFree(sendInfo); taosMemoryFree(sendInfo);
taosMemoryFree(pParam); taosMemoryFree(pParam);

26
source/common/src/tdatablock.c
View File

@ -1670,6 +1670,8 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
int32_t code = 0; int32_t code = 0;
dst->info = src->info; dst->info = src->info;
dst->info.pks[0].pData = NULL;
dst->info.pks[1].pData = NULL;
dst->info.rows = 0; dst->info.rows = 0;
dst->info.capacity = 0; dst->info.capacity = 0;
@ -1707,6 +1709,8 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
uint32_t cap = dst->info.capacity; uint32_t cap = dst->info.capacity;
dst->info = src->info; dst->info = src->info;
dst->info.pks[0].pData = NULL;
dst->info.pks[1].pData = NULL;
dst->info.capacity = cap; dst->info.capacity = cap;
return code; return code;
} }
@ -1737,6 +1741,8 @@ int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc) {
uint32_t cap = pDst->info.capacity; uint32_t cap = pDst->info.capacity;
pDst->info = pSrc->info; pDst->info = pSrc->info;
pDst->info.pks[0].pData = NULL;
pDst->info.pks[1].pData = NULL;
code = copyPkVal(&pDst->info, &pSrc->info); code = copyPkVal(&pDst->info, &pSrc->info);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
@ -1854,6 +1860,8 @@ int32_t blockCopyOneRow(const SSDataBlock* pDataBlock, int32_t rowIdx, SSDataBlo
} }
pBlock->info = pDataBlock->info; pBlock->info = pDataBlock->info;
pBlock->info.pks[0].pData = NULL;
pBlock->info.pks[1].pData = NULL;
pBlock->info.rows = 0; pBlock->info.rows = 0;
pBlock->info.capacity = 0; pBlock->info.capacity = 0;
@ -1916,20 +1924,20 @@ int32_t copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc) {
// prepare the pk buffer if needed // prepare the pk buffer if needed
SValue* p = &pDst->pks[0]; SValue* p = &pDst->pks[0];
p->type = pDst->pks[0].type; p->type = pSrc->pks[0].type;
p->pData = taosMemoryCalloc(1, pDst->pks[0].nData); p->pData = taosMemoryCalloc(1, pSrc->pks[0].nData);
QUERY_CHECK_NULL(p->pData, code, lino, _end, terrno); QUERY_CHECK_NULL(p->pData, code, lino, _end, terrno);
p->nData = pDst->pks[0].nData; p->nData = pSrc->pks[0].nData;
memcpy(p->pData, pDst->pks[0].pData, p->nData); memcpy(p->pData, pSrc->pks[0].pData, p->nData);
p = &pDst->pks[1]; p = &pDst->pks[1];
p->type = pDst->pks[1].type; p->type = pSrc->pks[1].type;
p->pData = taosMemoryCalloc(1, pDst->pks[1].nData); p->pData = taosMemoryCalloc(1, pSrc->pks[1].nData);
QUERY_CHECK_NULL(p->pData, code, lino, _end, terrno); QUERY_CHECK_NULL(p->pData, code, lino, _end, terrno);
p->nData = pDst->pks[1].nData; p->nData = pSrc->pks[1].nData;
memcpy(p->pData, pDst->pks[1].pData, p->nData); memcpy(p->pData, pSrc->pks[1].pData, p->nData);
_end: _end:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -1951,6 +1959,8 @@ int32_t createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData, SSDataB
} }
pDstBlock->info = pDataBlock->info; pDstBlock->info = pDataBlock->info;
pDstBlock->info.pks[0].pData = NULL;
pDstBlock->info.pks[1].pData = NULL;
pDstBlock->info.rows = 0; pDstBlock->info.rows = 0;
pDstBlock->info.capacity = 0; pDstBlock->info.capacity = 0;

14
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -83,12 +83,12 @@ extern "C" {
}\ }\
} }
#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", gtid:%s", __VA_ARGS__, buf);}} #define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", gtid:%s", __VA_ARGS__, buf);}} #define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", gtid:%s", __VA_ARGS__, buf);}} #define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", gtid:%s", __VA_ARGS__, buf);}} #define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", gtid:%s", __VA_ARGS__, buf);}} #define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", gtid:%s", __VA_ARGS__, buf);}} #define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", QID:%s", __VA_ARGS__, buf);}}
// clang-format on // clang-format on
@ -208,7 +208,7 @@ void dmGetMonitorSystemInfo(SMonSysInfo *pInfo);
int32_t dmReadFile(const char *path, const char *name, bool *pDeployed); int32_t dmReadFile(const char *path, const char *name, bool *pDeployed);
int32_t dmWriteFile(const char *path, const char *name, bool deployed); int32_t dmWriteFile(const char *path, const char *name, bool deployed);
int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile); int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile);
//int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile); // int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile);
// dmodule.c // dmodule.c
int32_t dmInitDndInfo(SDnodeData *pData); int32_t dmInitDndInfo(SDnodeData *pData);

14
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -41,12 +41,12 @@ extern "C" {
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
#define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", gtid:%s", __VA_ARGS__, buf);}} #define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", QID:%s", __VA_ARGS__, buf);}}
#define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", gtid:%s", __VA_ARGS__, buf);}} #define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", QID:%s", __VA_ARGS__, buf);}}
#define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", gtid:%s", __VA_ARGS__, buf);}} #define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", QID:%s", __VA_ARGS__, buf);}}
#define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", gtid:%s", __VA_ARGS__, buf);}} #define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", QID:%s", __VA_ARGS__, buf);}}
#define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", gtid:%s", __VA_ARGS__, buf);}} #define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", QID:%s", __VA_ARGS__, buf);}}
#define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);}} #define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", QID:%s", __VA_ARGS__, buf);}}
// clang-format on // clang-format on
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
@ -54,7 +54,7 @@ extern "C" {
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
typedef int32_t (*MndMsgFp)(SRpcMsg *pMsg); typedef int32_t (*MndMsgFp)(SRpcMsg *pMsg);
typedef int32_t (*MndMsgFpExt)(SRpcMsg *pMsg, SQueueInfo* pInfo); typedef int32_t (*MndMsgFpExt)(SRpcMsg *pMsg, SQueueInfo *pInfo);
typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef int32_t (*MndInitFp)(SMnode *pMnode);
typedef void (*MndCleanupFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode);
typedef int32_t (*ShowRetrieveFp)(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); typedef int32_t (*ShowRetrieveFp)(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);

2
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1774,7 +1774,7 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *
int32_t nCols) { int32_t nCols) {
// if (pColCmpr == NULL || colName == NULL) return -1; // if (pColCmpr == NULL || colName == NULL) return -1;
ASSERT(taosArrayGetSize(pField) == nCols); if (taosArrayGetSize(pField) != nCols) return TSDB_CODE_FAILED;
TAOS_FIELD *p = taosArrayGet(pField, 0); TAOS_FIELD *p = taosArrayGet(pField, 0);
int32_t code = 0; int32_t code = 0;

10
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -52,6 +52,8 @@ static void sdbResetData(SSdb *pSdb) {
SHashObj *hash = pSdb->hashObjs[i]; SHashObj *hash = pSdb->hashObjs[i];
if (hash == NULL) continue; if (hash == NULL) continue;
sdbWriteLock(pSdb, i);
SSdbRow **ppRow = taosHashIterate(hash, NULL); SSdbRow **ppRow = taosHashIterate(hash, NULL);
while (ppRow != NULL) { while (ppRow != NULL) {
SSdbRow *pRow = *ppRow; SSdbRow *pRow = *ppRow;
@ -60,15 +62,13 @@ static void sdbResetData(SSdb *pSdb) {
sdbFreeRow(pSdb, pRow, true); sdbFreeRow(pSdb, pRow, true);
ppRow = taosHashIterate(hash, ppRow); ppRow = taosHashIterate(hash, ppRow);
} }
}
for (ESdbType i = 0; i < SDB_MAX; ++i) {
SHashObj *hash = pSdb->hashObjs[i];
if (hash == NULL) continue;
taosHashClear(pSdb->hashObjs[i]); taosHashClear(pSdb->hashObjs[i]);
pSdb->tableVer[i] = 0; pSdb->tableVer[i] = 0;
pSdb->maxId[i] = 0; pSdb->maxId[i] = 0;
sdbUnLock(pSdb, i);
mInfo("sdb:%s is reset", sdbTableName(i)); mInfo("sdb:%s is reset", sdbTableName(i));
} }

12
source/dnode/vnode/src/inc/vnd.h
View File

@ -32,12 +32,12 @@ extern "C" {
#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", DEBUG_DEBUG, vDebugFlag, __VA_ARGS__); }} while(0) #define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", DEBUG_DEBUG, vDebugFlag, __VA_ARGS__); }} while(0)
#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", DEBUG_TRACE, vDebugFlag, __VA_ARGS__); }} while(0) #define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", DEBUG_TRACE, vDebugFlag, __VA_ARGS__); }} while(0)
#define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
// clang-format on // clang-format on

15
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -101,20 +101,6 @@ typedef struct SQueryNode SQueryNode;
#define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME "vnode.json"
#define VND_INFO_FNAME_TMP "vnode_tmp.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json"
#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count"
#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"
#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id"
#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep"
#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id"
#define VNODE_METRIC_TAG_NAME_USERNAME "username"
#define VNODE_METRIC_TAG_NAME_RESULT "result"
#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows"
// #define VNODE_METRIC_TAG_VALUE_INSERT "insert"
// #define VNODE_METRIC_TAG_VALUE_DELETE "delete"
// vnd.h // vnd.h
typedef int32_t (*_query_reseek_func_t)(void* pQHandle); typedef int32_t (*_query_reseek_func_t)(void* pQHandle);
struct SQueryNode { struct SQueryNode {
@ -452,7 +438,6 @@ typedef struct SVMonitorObj {
char strClusterId[TSDB_CLUSTER_ID_LEN]; char strClusterId[TSDB_CLUSTER_ID_LEN];
char strDnodeId[TSDB_NODE_ID_LEN]; char strDnodeId[TSDB_NODE_ID_LEN];
char strVgId[TSDB_VGROUP_ID_LEN]; char strVgId[TSDB_VGROUP_ID_LEN];
taos_counter_t* insertCounter;
} SVMonitorObj; } SVMonitorObj;
typedef struct { typedef struct {

1
source/dnode/vnode/src/meta/metaCache.c
View File

@ -512,7 +512,6 @@ static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid,
buf[0] = (uint64_t)pHashMap; buf[0] = (uint64_t)pHashMap;
buf[1] = suid; buf[1] = suid;
setMD5DigestInKey(buf, key, keyLen); setMD5DigestInKey(buf, key, keyLen);
ASSERT(keyLen == sizeof(uint64_t) * 2);
} }
int32_t metaGetCachedTableUidList(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, int32_t metaGetCachedTableUidList(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1,

8
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -275,10 +275,10 @@ void metaPauseTbCursor(SMTbCursor *pTbCur) {
int32_t metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first, int8_t move) { int32_t metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first, int8_t move) {
int32_t code = 0; int32_t code = 0;
int32_t lino; int32_t lino;
int8_t locked = 0; int8_t locked = 0;
if (pTbCur->paused) { if (pTbCur->paused) {
metaReaderDoInit(&pTbCur->mr, pTbCur->pMeta, META_READER_LOCK); metaReaderDoInit(&pTbCur->mr, pTbCur->pMeta, META_READER_LOCK);
locked = 1; locked = 1;
code = tdbTbcOpen(((SMeta *)pTbCur->pMeta)->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); code = tdbTbcOpen(((SMeta *)pTbCur->pMeta)->pUidIdx, (TBC **)&pTbCur->pDbc, NULL);
if (code != 0) { if (code != 0) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -673,7 +673,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv
} }
} }
if (ASSERTS(sver > 0, "failed to get table schema version: %d", sver)) { if (!(sver > 0)) {
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
goto _exit; goto _exit;
} }
@ -1608,8 +1608,6 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t
metaULock(pVnodeObj->pMeta); metaULock(pVnodeObj->pMeta);
if (numOfTables) *numOfTables = state.ctbNum; if (numOfTables) *numOfTables = state.ctbNum;
if (numOfCols) *numOfCols = state.colNum; if (numOfCols) *numOfCols = state.colNum;
ASSERTS(state.colNum > 0, "vgId:%d, suid:%" PRIi64 " nCols:%d <= 0 in metaCache", TD_VID(pVnodeObj), uid,
state.colNum);
goto _exit; goto _exit;
} }

14
source/dnode/vnode/src/tq/tq.c
View File

@ -173,7 +173,7 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) {
dataRsp.common.blockNum = 0; dataRsp.common.blockNum = 0;
char buf[TSDB_OFFSET_LEN] = {0}; char buf[TSDB_OFFSET_LEN] = {0};
(void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset); (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset);
tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, reqId:0x%" PRIx64, req.consumerId, vgId, buf, tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, QID:0x%" PRIx64, req.consumerId, vgId, buf,
req.reqId); req.reqId);
code = tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); code = tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
@ -193,7 +193,7 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq*
(void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset); (void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset);
(void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset); (void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset);
tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, QID:0x%" PRIx64,
vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId); vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId);
return tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); return tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever);
@ -421,7 +421,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
char buf[TSDB_OFFSET_LEN] = {0}; char buf[TSDB_OFFSET_LEN] = {0};
(void)tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset); (void)tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, QID:0x%" PRIx64,
consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId);
code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg);
@ -1216,13 +1216,13 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
} }
if (req.mndTrigger) { if (req.mndTrigger) {
tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ", pTask->id.idStr, tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ",
vgId, pTask->info.taskLevel, req.checkpointId, req.transId); pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId);
} else { } else {
const char* pPrevStatus = streamTaskGetStatusStr(streamTaskGetPrevStatus(pTask)); const char* pPrevStatus = streamTaskGetStatusStr(streamTaskGetPrevStatus(pTask));
tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64
", transId:%d after transfer-state, prev status:%s", ", transId:%d after transfer-state, prev status:%s",
pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId, pPrevStatus); pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId, pPrevStatus);
} }
code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask); code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask);

145
source/dnode/vnode/src/tq/tqRead.c
View File

@ -74,7 +74,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
pCreateReq = req.pReqs + iReq; pCreateReq = req.pReqs + iReq;
if (pCreateReq->type == TSDB_CHILD_TABLE && pCreateReq->ctb.suid == tbSuid) { if (pCreateReq->type == TSDB_CHILD_TABLE && pCreateReq->ctb.suid == tbSuid) {
reqNew.nReqs++; reqNew.nReqs++;
if (taosArrayPush(reqNew.pArray, pCreateReq) == NULL){ if (taosArrayPush(reqNew.pArray, pCreateReq) == NULL) {
taosArrayDestroy(reqNew.pArray); taosArrayDestroy(reqNew.pArray);
tDeleteSVCreateTbBatchReq(&req); tDeleteSVCreateTbBatchReq(&req);
goto end; goto end;
@ -155,7 +155,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
pDropReq = req.pReqs + iReq; pDropReq = req.pReqs + iReq;
if (pDropReq->suid == tbSuid) { if (pDropReq->suid == tbSuid) {
reqNew.nReqs++; reqNew.nReqs++;
if (taosArrayPush(reqNew.pArray, pDropReq) == NULL){ if (taosArrayPush(reqNew.pArray, pDropReq) == NULL) {
taosArrayDestroy(reqNew.pArray); taosArrayDestroy(reqNew.pArray);
goto end; goto end;
} }
@ -214,12 +214,12 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t
while (offset <= appliedVer) { while (offset <= appliedVer) {
if (walFetchHead(pHandle->pWalReader, offset) < 0) { if (walFetchHead(pHandle->pWalReader, offset) < 0) {
tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64
", no more log to return, reqId:0x%" PRIx64 " 0x%" PRIx64, ", no more log to return, QID:0x%" PRIx64 " 0x%" PRIx64,
pHandle->consumerId, pHandle->epoch, vgId, offset, reqId, id); pHandle->consumerId, pHandle->epoch, vgId, offset, reqId, id);
goto END; goto END;
} }
tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64 " 0x%" PRIx64, tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, QID:0x%" PRIx64 " 0x%" PRIx64,
vgId, pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id); vgId, pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id);
if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) { if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) {
@ -261,10 +261,10 @@ END:
bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; } bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; }
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid){ void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
bool ret = false; bool ret = false;
SSchemaWrapper *schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1); SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){ if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true; ret = true;
} }
tDeleteSchemaWrapper(schema); tDeleteSchemaWrapper(schema);
@ -487,7 +487,7 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) {
(pReader->nextBlk + 1), numOfBlocks, idstr); (pReader->nextBlk + 1), numOfBlocks, idstr);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if (pSubmitTbData == NULL){ if (pSubmitTbData == NULL) {
return false; return false;
} }
if (pReader->tbIdHash == NULL) { if (pReader->tbIdHash == NULL) {
@ -640,9 +640,9 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol
int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) { int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) {
tqTrace("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); tqTrace("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk);
int32_t code = 0; int32_t code = 0;
int32_t line = 0; int32_t line = 0;
STSchema* pTSchema = NULL; STSchema* pTSchema = NULL;
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++);
TSDB_CHECK_NULL(pSubmitTbData, code, line, END, terrno); TSDB_CHECK_NULL(pSubmitTbData, code, line, END, terrno);
SSDataBlock* pBlock = pReader->pResBlock; SSDataBlock* pBlock = pReader->pResBlock;
@ -717,10 +717,11 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
continue; continue;
} }
SColData* pCol = taosArrayGet(pCols, sourceIdx); SColData* pCol = taosArrayGet(pCols, sourceIdx);
TSDB_CHECK_NULL(pCol, code, line, END, terrno); TSDB_CHECK_NULL(pCol, code, line, END, terrno);
SColVal colVal = {0}; SColVal colVal = {0};
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual, sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId); tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual,
sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
if (pCol->cid < pColData->info.colId) { if (pCol->cid < pColData->info.colId) {
sourceIdx++; sourceIdx++;
} else if (pCol->cid == pColData->info.colId) { } else if (pCol->cid == pColData->info.colId) {
@ -743,7 +744,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
TSDB_CHECK_NULL(pTSchema, code, line, END, terrno); TSDB_CHECK_NULL(pTSchema, code, line, END, terrno);
for (int32_t i = 0; i < numOfRows; i++) { for (int32_t i = 0; i < numOfRows; i++) {
SRow* pRow = taosArrayGetP(pRows, i); SRow* pRow = taosArrayGetP(pRows, i);
TSDB_CHECK_NULL(pRow, code, line, END, terrno); TSDB_CHECK_NULL(pRow, code, line, END, terrno);
int32_t sourceIdx = 0; int32_t sourceIdx = 0;
for (int32_t j = 0; j < colActual; j++) { for (int32_t j = 0; j < colActual; j++) {
@ -770,44 +771,43 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
} }
} }
} }
} }
END: END:
if (code != 0){ if (code != 0) {
tqError("tqRetrieveDataBlock failed, line:%d, code:%d", line, code); tqError("tqRetrieveDataBlock failed, line:%d, code:%d", line, code);
} }
taosMemoryFreeClear(pTSchema); taosMemoryFreeClear(pTSchema);
return code; return code;
} }
#define PROCESS_VAL \ #define PROCESS_VAL \
if (curRow == 0) {\ if (curRow == 0) { \
assigned[j] = !COL_VAL_IS_NONE(&colVal);\ assigned[j] = !COL_VAL_IS_NONE(&colVal); \
buildNew = true;\ buildNew = true; \
} else {\ } else { \
bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal);\ bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal); \
if (currentRowAssigned != assigned[j]) {\ if (currentRowAssigned != assigned[j]) { \
assigned[j] = currentRowAssigned;\ assigned[j] = currentRowAssigned; \
buildNew = true;\ buildNew = true; \
}\ } \
} }
#define SET_DATA \ #define SET_DATA \
if (colVal.cid < pColData->info.colId) {\ if (colVal.cid < pColData->info.colId) { \
sourceIdx++;\ sourceIdx++; \
} else if (colVal.cid == pColData->info.colId) {\ } else if (colVal.cid == pColData->info.colId) { \
TQ_ERR_GO_TO_END(doSetVal(pColData, curRow - lastRow, &colVal));\ TQ_ERR_GO_TO_END(doSetVal(pColData, curRow - lastRow, &colVal)); \
sourceIdx++;\ sourceIdx++; \
targetIdx++;\ targetIdx++; \
} }
static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas,
SArray* schemas, SSchemaWrapper* pSchemaWrapper, char* assigned, SSchemaWrapper* pSchemaWrapper, char* assigned, int32_t numOfRows, int32_t curRow,
int32_t numOfRows, int32_t curRow, int32_t* lastRow){ int32_t* lastRow) {
int32_t code = 0; int32_t code = 0;
SSchemaWrapper* pSW = NULL; SSchemaWrapper* pSW = NULL;
SSDataBlock* block = NULL; SSDataBlock* block = NULL;
if (taosArrayGetSize(blocks) > 0) { if (taosArrayGetSize(blocks) > 0) {
SSDataBlock* pLastBlock = taosArrayGetLast(blocks); SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
TQ_NULL_GO_TO_END(pLastBlock); TQ_NULL_GO_TO_END(pLastBlock);
@ -839,33 +839,34 @@ END:
taosMemoryFree(block); taosMemoryFree(block);
return code; return code;
} }
static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas){ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) {
int32_t code = 0; int32_t code = 0;
int32_t curRow = 0; int32_t curRow = 0;
int32_t lastRow = 0; int32_t lastRow = 0;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
TQ_NULL_GO_TO_END(assigned); TQ_NULL_GO_TO_END(assigned);
SArray* pCols = pSubmitTbData->aCol; SArray* pCols = pSubmitTbData->aCol;
SColData* pCol = taosArrayGet(pCols, 0); SColData* pCol = taosArrayGet(pCols, 0);
TQ_NULL_GO_TO_END(pCol); TQ_NULL_GO_TO_END(pCol);
int32_t numOfRows = pCol->nVal; int32_t numOfRows = pCol->nVal;
int32_t numOfCols = taosArrayGetSize(pCols); int32_t numOfCols = taosArrayGetSize(pCols);
for (int32_t i = 0; i < numOfRows; i++) { for (int32_t i = 0; i < numOfRows; i++) {
bool buildNew = false; bool buildNew = false;
for (int32_t j = 0; j < numOfCols; j++) { for (int32_t j = 0; j < numOfCols; j++) {
pCol = taosArrayGet(pCols, j); pCol = taosArrayGet(pCols, j);
TQ_NULL_GO_TO_END(pCol); TQ_NULL_GO_TO_END(pCol);
SColVal colVal = {0}; SColVal colVal = {0};
tColDataGetValue(pCol, i, &colVal); tColDataGetValue(pCol, i, &colVal);
PROCESS_VAL PROCESS_VAL
} }
if (buildNew) { if (buildNew) {
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, curRow, &lastRow)); TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows,
curRow, &lastRow));
} }
SSDataBlock* pBlock = taosArrayGetLast(blocks); SSDataBlock* pBlock = taosArrayGetLast(blocks);
@ -882,7 +883,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
TQ_NULL_GO_TO_END(pCol); TQ_NULL_GO_TO_END(pCol);
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
TQ_NULL_GO_TO_END(pColData); TQ_NULL_GO_TO_END(pColData);
SColVal colVal = {0}; SColVal colVal = {0};
tColDataGetValue(pCol, i, &colVal); tColDataGetValue(pCol, i, &colVal);
SET_DATA SET_DATA
} }
@ -890,30 +891,30 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
curRow++; curRow++;
} }
SSDataBlock* pLastBlock = taosArrayGetLast(blocks); SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow; pLastBlock->info.rows = curRow - lastRow;
END: END:
taosMemoryFree(assigned); taosMemoryFree(assigned);
return code; return code;
} }
int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas){ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) {
int32_t code = 0; int32_t code = 0;
STSchema* pTSchema = NULL; STSchema* pTSchema = NULL;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
TQ_NULL_GO_TO_END(assigned); TQ_NULL_GO_TO_END(assigned);
int32_t curRow = 0; int32_t curRow = 0;
int32_t lastRow = 0; int32_t lastRow = 0;
SArray* pRows = pSubmitTbData->aRowP; SArray* pRows = pSubmitTbData->aRowP;
int32_t numOfRows = taosArrayGetSize(pRows); int32_t numOfRows = taosArrayGetSize(pRows);
pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version);
for (int32_t i = 0; i < numOfRows; i++) { for (int32_t i = 0; i < numOfRows; i++) {
bool buildNew = false; bool buildNew = false;
SRow* pRow = taosArrayGetP(pRows, i); SRow* pRow = taosArrayGetP(pRows, i);
TQ_NULL_GO_TO_END(pRow); TQ_NULL_GO_TO_END(pRow);
for (int32_t j = 0; j < pTSchema->numOfCols; j++) { for (int32_t j = 0; j < pTSchema->numOfCols; j++) {
@ -923,7 +924,8 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
} }
if (buildNew) { if (buildNew) {
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, curRow, &lastRow)); TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows,
curRow, &lastRow));
} }
SSDataBlock* pBlock = taosArrayGetLast(blocks); SSDataBlock* pBlock = taosArrayGetLast(blocks);
@ -937,7 +939,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
int32_t colActual = blockDataGetNumOfCols(pBlock); int32_t colActual = blockDataGetNumOfCols(pBlock);
while (targetIdx < colActual) { while (targetIdx < colActual) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
SColVal colVal = {0}; SColVal colVal = {0};
TQ_ERR_GO_TO_END(tRowGet(pRow, pTSchema, sourceIdx, &colVal)); TQ_ERR_GO_TO_END(tRowGet(pRow, pTSchema, sourceIdx, &colVal));
SET_DATA SET_DATA
} }
@ -945,7 +947,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
curRow++; curRow++;
} }
SSDataBlock* pLastBlock = taosArrayGetLast(blocks); SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow; pLastBlock->info.rows = curRow - lastRow;
END: END:
taosMemoryFreeClear(pTSchema); taosMemoryFreeClear(pTSchema);
@ -955,10 +957,10 @@ END:
int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) {
tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk); tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
SSDataBlock* block = NULL; SSDataBlock* block = NULL;
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if(pSubmitTbData == NULL){ if (pSubmitTbData == NULL) {
return terrno; return terrno;
} }
pReader->nextBlk++; pReader->nextBlk++;
@ -1039,7 +1041,7 @@ bool tqCurrentBlockConsumed(const STqReader* pReader) { return pReader->msg.msgS
void tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { void tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
if (pKey && taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)) != 0){ if (pKey && taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)) != 0) {
tqError("failed to remove table uid:%" PRId64 " from hash", *pKey); tqError("failed to remove table uid:%" PRId64 " from hash", *pKey);
} }
} }
@ -1069,7 +1071,8 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
int32_t sz = taosArrayGetSize(tbUidList); int32_t sz = taosArrayGetSize(tbUidList);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
int64_t* tbUid = (int64_t*)taosArrayGet(tbUidList, i); int64_t* tbUid = (int64_t*)taosArrayGet(tbUidList, i);
if (tbUid && taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, tbUid, sizeof(int64_t), NULL, 0) != 0){ if (tbUid &&
taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, tbUid, sizeof(int64_t), NULL, 0) != 0) {
tqError("failed to add table uid:%" PRId64 " to hash", *tbUid); tqError("failed to add table uid:%" PRId64 " to hash", *tbUid);
continue; continue;
} }

22
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -92,7 +92,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
char formatBuf[TSDB_OFFSET_LEN] = {0}; char formatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal); tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal);
tqDebug("tmq poll: consumer:0x%" PRIx64 tqDebug("tmq poll: consumer:0x%" PRIx64
", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64, ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. QID:0x%" PRIx64,
consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId);
return 0; return 0;
} else { } else {
@ -117,7 +117,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1); tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1);
code = tqInitDataRsp(&dataRsp.common, *pOffsetVal); code = tqInitDataRsp(&dataRsp.common, *pOffsetVal);
if (code != 0){ if (code != 0) {
return code; return code;
} }
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId,
@ -145,7 +145,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
terrno = 0; terrno = 0;
SMqDataRsp dataRsp = {0}; SMqDataRsp dataRsp = {0};
int code = tqInitDataRsp(&dataRsp.common, *pOffset); int code = tqInitDataRsp(&dataRsp.common, *pOffset);
if (code != 0) { if (code != 0) {
goto end; goto end;
} }
@ -176,7 +176,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
end : { end : {
char buf[TSDB_OFFSET_LEN] = {0}; char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset); tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, QID:0x%" PRIx64
" code:%d", " code:%d",
consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code); consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code);
tDeleteMqDataRsp(&dataRsp); tDeleteMqDataRsp(&dataRsp);
@ -206,10 +206,10 @@ static void tDeleteCommon(void* parm) {}
static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, STqOffsetVal* offset) { SRpcMsg* pMsg, STqOffsetVal* offset) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
STaosxRsp taosxRsp = {0}; STaosxRsp taosxRsp = {0};
SMqBatchMetaRsp btMetaRsp = {0}; SMqBatchMetaRsp btMetaRsp = {0};
int32_t code = 0; int32_t code = 0;
TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp.common, *offset)); TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp.common, *offset));
if (offset->type != TMQ_OFFSET__LOG) { if (offset->type != TMQ_OFFSET__LOG) {
@ -610,7 +610,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void*
SColumnInfoData* pUidCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX); SColumnInfoData* pUidCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX);
TSDB_CHECK_NULL(pUidCol, code, line, END, terrno) TSDB_CHECK_NULL(pUidCol, code, line, END, terrno)
int64_t* pUid = taosArrayGet(pRes->uidList, i); int64_t* pUid = taosArrayGet(pRes->uidList, i);
code = colDataSetVal(pUidCol, i, (const char*)pUid, false); code = colDataSetVal(pUidCol, i, (const char*)pUid, false);
TSDB_CHECK_CODE(code, line, END); TSDB_CHECK_CODE(code, line, END);
void* tmp = taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX); void* tmp = taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX);
@ -642,7 +642,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void*
} }
END: END:
if (code != 0){ if (code != 0) {
tqError("failed to extract delete data block, line:%d code:%d", line, code); tqError("failed to extract delete data block, line:%d code:%d", line, code);
} }
tDecoderClear(pCoder); tDecoderClear(pCoder);
@ -672,7 +672,7 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i);
if (pId == NULL){ if (pId == NULL) {
continue; continue;
} }
if (pId->streamId != streamId) { if (pId->streamId != streamId) {

73
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -207,7 +207,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
updated = streamTaskUpdateEpsetInfo(pTask, req.pNodeList); updated = streamTaskUpdateEpsetInfo(pTask, req.pNodeList);
// send the checkpoint-source-rsp for source task to end the checkpoint trans in mnode // send the checkpoint-source-rsp for source task to end the checkpoint trans in mnode
(void) streamTaskSendCheckpointsourceRsp(pTask); (void)streamTaskSendCheckpointsourceRsp(pTask);
streamTaskResetStatus(pTask); streamTaskResetStatus(pTask);
streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr); streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr);
@ -303,7 +303,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return rsp.code; // always return true return rsp.code; // always return true
} }
int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -324,7 +324,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId); tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId);
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask && (code == 0)) { if (pTask && (code == 0)) {
SRpcMsg rsp = {.info = pMsg->info, .code = 0}; SRpcMsg rsp = {.info = pMsg->info, .code = 0};
if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) { if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) {
@ -384,7 +384,7 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
pRsp->downstreamTaskId, pRsp->downstreamNodeId); pRsp->downstreamTaskId, pRsp->downstreamNodeId);
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
if (pTask && (code == 0)) { if (pTask && (code == 0)) {
code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code); code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -482,8 +482,8 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", tqDebug("tq task:0x%x (vgId:%d) recv check rsp(QID:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId,
rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);
if (!isLeader) { if (!isLeader) {
tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId, tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId,
@ -681,7 +681,8 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
if (code) { if (code) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId); tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId,
(int32_t)hTaskId.taskId);
} }
} }
@ -701,7 +702,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
return 0; // always return success return 0; // always return success
} }
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) { int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) {
@ -793,23 +794,23 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
if (type == STREAM_EXEC_T_START_ONE_TASK) { if (type == STREAM_EXEC_T_START_ONE_TASK) {
(void) streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); (void)streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
return 0; return 0;
} else if (type == STREAM_EXEC_T_START_ALL_TASKS) { } else if (type == STREAM_EXEC_T_START_ALL_TASKS) {
(void) streamMetaStartAllTasks(pMeta); (void)streamMetaStartAllTasks(pMeta);
return 0; return 0;
} else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) { } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) {
(void) restartStreamTasks(pMeta, isLeader); (void)restartStreamTasks(pMeta, isLeader);
return 0; return 0;
} else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) { } else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) {
(void) streamMetaStopAllTasks(pMeta); (void)streamMetaStopAllTasks(pMeta);
return 0; return 0;
} else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) { } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId); int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
return code; return code;
} else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while } else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL && (code == 0)) { if (pTask != NULL && (code == 0)) {
char* pStatus = NULL; char* pStatus = NULL;
@ -831,13 +832,13 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
} }
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed
char* p = NULL; char* p = NULL;
if (streamTaskReadyToRun(pTask, &p)) { if (streamTaskReadyToRun(pTask, &p)) {
tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId,
pTask->id.idStr, p, pTask->chkInfo.nextProcessVer); pTask->id.idStr, p, pTask->chkInfo.nextProcessVer);
(void) streamExecTask(pTask); (void)streamExecTask(pTask);
} else { } else {
int8_t status = streamTaskSetSchedStatusInactive(pTask); int8_t status = streamTaskSetSchedStatusInactive(pTask);
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
@ -900,7 +901,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL || (code != 0)) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->taskId); pMeta->vgId, pReq->taskId);
@ -924,9 +925,9 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
streamTaskSetStatusReady(pTask); streamTaskSetStatusReady(pTask);
} else if (pState.state == TASK_STATUS__UNINIT) { } else if (pState.state == TASK_STATUS__UNINIT) {
// tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr); // tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
// ASSERT(pTask->status.downstreamReady == 0); // ASSERT(pTask->status.downstreamReady == 0);
// tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId); // tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
} else { } else {
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
@ -942,7 +943,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont; SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont;
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
if (pTask == NULL || (code != 0)) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
" from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
@ -958,7 +959,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
pTask->id.idStr, (int32_t)pReq->downstreamTaskId); pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_STREAM_TASK_IVLD_STATUS); TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return code; return code;
} }
@ -996,7 +997,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
pTask->id.idStr, recv, total); pTask->id.idStr, recv, total);
} }
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS); TSDB_CODE_ACTION_IN_PROGRESS);
} }
} else { // upstream not recv the checkpoint-source/trigger till now } else { // upstream not recv the checkpoint-source/trigger till now
ASSERT(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT); ASSERT(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT);
@ -1005,7 +1006,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"upstream sending checkpoint-source/trigger", "upstream sending checkpoint-source/trigger",
pTask->id.idStr); pTask->id.idStr);
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS); TSDB_CODE_ACTION_IN_PROGRESS);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -1016,7 +1017,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
if (pTask == NULL || (code != 0)) { if (pTask == NULL || (code != 0)) {
tqError( tqError(
"vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
@ -1038,7 +1039,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL || (code != 0)) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
pReq->taskId); pReq->taskId);
@ -1122,7 +1123,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL || (code != 0)) { if (pTask == NULL || (code != 0)) {
tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId); tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS; return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
@ -1169,13 +1170,15 @@ int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { ret
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) {
return doProcessDummyRspMsg(pMeta, pMsg);
}
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
if (pTask == NULL || (code != 0)) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped",
pRsp->downstreamNodeId, pRsp->downstreamTaskId); pRsp->downstreamNodeId, pRsp->downstreamTaskId);
@ -1211,8 +1214,9 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL || (code != 0)) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", tqError(
pMeta->vgId, req.taskId); "vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId);
(void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); (void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
return code; return code;
} }
@ -1221,7 +1225,8 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
if (req.startTs < pTask->execInfo.created) { if (req.startTs < pTask->execInfo.created) {
tqWarn("s-task:%s vgId:%d create time:%" PRId64 " recv expired consensus checkpointId:%" PRId64 tqWarn("s-task:%s vgId:%d create time:%" PRId64 " recv expired consensus checkpointId:%" PRId64
" from task createTs:%" PRId64 " < task createTs:%" PRId64 ", discard", " from task createTs:%" PRId64 " < task createTs:%" PRId64 ", discard",
pTask->id.idStr, pMeta->vgId, pTask->execInfo.created, req.checkpointId, req.startTs, pTask->execInfo.created); pTask->id.idStr, pMeta->vgId, pTask->execInfo.created, req.checkpointId, req.startTs,
pTask->execInfo.created);
streamMetaAddFailedTaskSelf(pTask, now); streamMetaAddFailedTaskSelf(pTask, now);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1234,15 +1239,15 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId); ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId);
if (pTask->chkInfo.consensusTransId >= req.transId) { if (pTask->chkInfo.consensusTransId >= req.transId) {
tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", pTask->id.idStr, vgId,
pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId); pTask->chkInfo.consensusTransId, req.transId);
streamMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (pTask->chkInfo.checkpointId != req.checkpointId) { if (pTask->chkInfo.checkpointId != req.checkpointId) {
tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64" transId:%d", pTask->id.idStr, vgId, tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64 " transId:%d", pTask->id.idStr, vgId,
pTask->chkInfo.checkpointId, req.checkpointId, req.transId); pTask->chkInfo.checkpointId, req.checkpointId, req.transId);
pTask->chkInfo.checkpointId = req.checkpointId; pTask->chkInfo.checkpointId = req.checkpointId;
tqSetRestoreVersionInfo(pTask); tqSetRestoreVersionInfo(pTask);

3
source/dnode/vnode/src/vnd/vnodeModule.c
View File

@ -14,6 +14,7 @@
*/ */
#include "cos.h" #include "cos.h"
#include "monitor.h"
#include "vnd.h" #include "vnd.h"
static volatile int32_t VINIT = 0; static volatile int32_t VINIT = 0;
@ -26,6 +27,8 @@ int vnodeInit(int nthreads, StopDnodeFp stopDnodeFp) {
TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads)); TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads));
TAOS_CHECK_RETURN(walInit(stopDnodeFp)); TAOS_CHECK_RETURN(walInit(stopDnodeFp));
monInitVnode();
return 0; return 0;
} }

18
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -494,24 +494,6 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%" PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId); snprintf(pVnode->monitor.strDnodeId, TSDB_NODE_ID_LEN, "%" PRId32, pVnode->config.syncCfg.nodeInfo[0].nodeId);
snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%" PRId32, pVnode->config.vgId); snprintf(pVnode->monitor.strVgId, TSDB_VGROUP_ID_LEN, "%" PRId32, pVnode->config.vgId);
if (tsEnableMonitor && pVnode->monitor.insertCounter == NULL) {
taos_counter_t *counter = NULL;
int32_t label_count = 7;
const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
VNODE_METRIC_TAG_NAME_RESULT};
counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels);
vInfo("vgId:%d, new metric:%p", TD_VID(pVnode), counter);
if (taos_collector_registry_register_metric(counter) == 1) {
(void)taos_counter_destroy(counter);
counter = taos_collector_registry_get_metric(VNODE_METRIC_SQL_COUNT);
vInfo("vgId:%d, get metric from registry:%p", TD_VID(pVnode), counter);
}
pVnode->monitor.insertCounter = counter;
vInfo("vgId:%d, succeed to set metric:%p", TD_VID(pVnode), counter);
}
return pVnode; return pVnode;
_err: _err:

8
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -15,6 +15,7 @@
#include "audit.h" #include "audit.h"
#include "cos.h" #include "cos.h"
#include "monitor.h"
#include "tencode.h" #include "tencode.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h" #include "tmsg.h"
@ -23,6 +24,8 @@
#include "vnode.h" #include "vnode.h"
#include "vnodeInt.h" #include "vnodeInt.h"
extern taos_counter_t *tsInsertCounter;
static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
@ -1903,7 +1906,8 @@ _exit:
(void)atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); (void)atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows);
(void)atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); (void)atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1);
if (tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0) { if (tsEnableMonitor && tsMonitorFqdn[0] != 0 && tsMonitorPort != 0 && pSubmitRsp->affectedRows > 0 &&
strlen(pOriginalMsg->info.conn.user) > 0 && tsInsertCounter != NULL) {
const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS,
pVnode->monitor.strClusterId, pVnode->monitor.strClusterId,
pVnode->monitor.strDnodeId, pVnode->monitor.strDnodeId,
@ -1911,7 +1915,7 @@ _exit:
pVnode->monitor.strVgId, pVnode->monitor.strVgId,
pOriginalMsg->info.conn.user, pOriginalMsg->info.conn.user,
"Success"}; "Success"};
(void)taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels); (void)taos_counter_add(tsInsertCounter, pSubmitRsp->affectedRows, sample_labels);
} }
if (code == 0) { if (code == 0) {

411
source/libs/catalog/src/ctgAsync.c
View File

File diff suppressed because it is too large Load Diff

76
source/libs/catalog/src/ctgRemote.c
View File

@ -57,13 +57,13 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
} }
for (int32_t i = 0; i < taskNum; ++i) { for (int32_t i = 0; i < taskNum; ++i) {
int32_t* taskId = taosArrayGet(cbParam->taskId, i); int32_t* taskId = taosArrayGet(cbParam->taskId, i);
if (NULL == taskId) { if (NULL == taskId) {
ctgError("taosArrayGet %d taskId failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->taskId)); ctgError("taosArrayGet %d taskId failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->taskId));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} }
int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i); int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i);
if (NULL == msgIdx) { if (NULL == msgIdx) {
ctgError("taosArrayGet %d msgIdx failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->msgIdx)); ctgError("taosArrayGet %d msgIdx failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->msgIdx));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
@ -80,7 +80,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
if (pRsp->msgIdx != *msgIdx) { if (pRsp->msgIdx != *msgIdx) {
ctgError("rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx); ctgError("rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx);
pRsp = &rsp; pRsp = &rsp;
pRsp->msgIdx = *msgIdx; pRsp->msgIdx = *msgIdx;
pRsp->reqType = -1; pRsp->reqType = -1;
@ -117,7 +117,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId,
pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs);
(void)(*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal (void)(*gCtgAsyncFps[pTask->type].handleRspFp)(
&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal
} }
CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs));
@ -420,12 +421,12 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
if (TDMT_VND_BATCH_META == cbParam->reqType || TDMT_MND_BATCH_META == cbParam->reqType) { if (TDMT_VND_BATCH_META == cbParam->reqType || TDMT_MND_BATCH_META == cbParam->reqType) {
CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode)); CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode));
} else { } else {
int32_t* taskId = taosArrayGet(cbParam->taskId, 0); int32_t* taskId = taosArrayGet(cbParam->taskId, 0);
if (NULL == taskId) { if (NULL == taskId) {
ctgError("taosArrayGet %d taskId failed, total:%d", 0, (int32_t)taosArrayGetSize(cbParam->taskId)); ctgError("taosArrayGet %d taskId failed, total:%d", 0, (int32_t)taosArrayGetSize(cbParam->taskId));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} }
SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId);
if (NULL == pTask) { if (NULL == pTask) {
ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks)); ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks));
@ -448,7 +449,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
ctgError("get task %d SCtgMsgCtx failed, taskType:%d", -1, pTask->type); ctgError("get task %d SCtgMsgCtx failed, taskType:%d", -1, pTask->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} }
pMsgCtx->pBatchs = pBatchs; pMsgCtx->pBatchs = pBatchs;
#endif #endif
@ -537,7 +538,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob,
CTG_ERR_JRET(code); CTG_ERR_JRET(code);
} }
ctgDebug("ctg req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType)); ctgDebug("ctg req msg sent, QID:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
@ -561,9 +562,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq->msgIdx, pTask->type); ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq->msgIdx, pTask->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} }
SHashObj* pBatchs = pMsgCtx->pBatchs; SHashObj* pBatchs = pMsgCtx->pBatchs;
SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId));
if (NULL == pBatch) { if (NULL == pBatch) {
newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg));
newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t));
@ -602,7 +603,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName));
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){ } else if (CTG_TASK_GET_TB_TSMA == pTask->type) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
@ -619,10 +620,11 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) { if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx,
(int32_t)taosArrayGetSize(pCtx->pFetches));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} }
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
if (NULL == pTbReq) { if (NULL == pTbReq) {
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
@ -678,7 +680,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName));
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){ } else if (CTG_TASK_GET_TB_TSMA == pTask->type) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
@ -692,22 +694,23 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
pName = ctx->pName; pName = ctx->pName;
} }
} else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) { } else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) { if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx,
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); (int32_t)taosArrayGetSize(pCtx->pFetches));
} CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); }
if (NULL == pTbReq) { STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); if (NULL == pTbReq) {
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
} CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); }
if (NULL == pName) { pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); if (NULL == pName) {
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables));
} CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
} else { } else {
ctgError("invalid vnode msgType %d", msgType); ctgError("invalid vnode msgType %d", msgType);
CTG_ERR_JRET(TSDB_CODE_APP_ERROR); CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
@ -1632,9 +1635,9 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName*
} }
int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* name, STableTSMAInfoRsp* out, int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* name, STableTSMAInfoRsp* out,
SCtgTaskReq* tReq, int32_t reqType) { SCtgTaskReq* tReq, int32_t reqType) {
char* msg = NULL; char* msg = NULL;
int32_t msgLen = 0; int32_t msgLen = 0;
SCtgTask* pTask = tReq ? tReq->pTask : NULL; SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
@ -1723,7 +1726,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c
#if CTG_BATCH_FETCH #if CTG_BATCH_FETCH
CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen)); CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen));
#else #else
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
(void)tNameGetFullDbName(pTbName, dbFName); (void)tNameGetFullDbName(pTbName, dbFName);
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) { if (NULL == pTaskId) {
@ -1734,7 +1737,8 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen)); CTG_RET(
ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen));
#endif #endif
} }

2
source/libs/executor/inc/tsort.h
View File

@ -66,7 +66,7 @@ typedef struct SMsortComparParam {
typedef struct SSortHandle SSortHandle; typedef struct SSortHandle SSortHandle;
typedef struct STupleHandle STupleHandle; typedef struct STupleHandle STupleHandle;
typedef SSDataBlock* (*_sort_fetch_block_fn_t)(void* param); typedef int32_t (*_sort_fetch_block_fn_t)(void* param, SSDataBlock** ppBlock);
typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* param); typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* param);
/** /**

1
source/libs/executor/src/exchangeoperator.c
View File

@ -521,6 +521,7 @@ void doDestroyExchangeOperatorInfo(void* param) {
int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param;
taosMemoryFreeClear(pMsg->pEpSet);
SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId);
if (pExchangeInfo == NULL) { if (pExchangeInfo == NULL) {
qWarn("failed to acquire exchange operator, since it may have been released, %p", pExchangeInfo); qWarn("failed to acquire exchange operator, since it may have been released, %p", pExchangeInfo);

6
source/libs/executor/src/mergeoperator.c
View File

@ -65,10 +65,10 @@ static SSDataBlock* doNonSortMerge1(SOperatorInfo* pOperator);
static SSDataBlock* doColsMerge1(SOperatorInfo* pOperator); static SSDataBlock* doColsMerge1(SOperatorInfo* pOperator);
static int32_t doColsMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock); static int32_t doColsMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock);
SSDataBlock* sortMergeloadNextDataBlock(void* param) { int32_t sortMergeloadNextDataBlock(void* param, SSDataBlock** ppBlock) {
SOperatorInfo* pOperator = (SOperatorInfo*)param; SOperatorInfo* pOperator = (SOperatorInfo*)param;
SSDataBlock* pBlock = pOperator->fpSet.getNextFn(pOperator); *ppBlock = pOperator->fpSet.getNextFn(pOperator);
return pBlock; return TSDB_CODE_SUCCESS;
} }
int32_t openSortMergeOperator(SOperatorInfo* pOperator) { int32_t openSortMergeOperator(SOperatorInfo* pOperator) {

19
source/libs/executor/src/scanoperator.c
View File

@ -5414,7 +5414,7 @@ static void doGetBlockForTableMergeScan(SOperatorInfo* pOperator, bool* pFinishe
return; return;
} }
static SSDataBlock* getBlockForTableMergeScan(void* param) { static int32_t getBlockForTableMergeScan(void* param, SSDataBlock** ppBlock) {
STableMergeScanSortSourceParam* source = param; STableMergeScanSortSourceParam* source = param;
SOperatorInfo* pOperator = source->pOperator; SOperatorInfo* pOperator = source->pOperator;
@ -5422,6 +5422,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSDataBlock* pBlock = NULL; SSDataBlock* pBlock = NULL;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
int32_t code = TSDB_CODE_SUCCESS;
while (true) { while (true) {
if (pInfo->rtnNextDurationBlocks) { if (pInfo->rtnNextDurationBlocks) {
@ -5456,10 +5457,11 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
if (pInfo->bNextDurationBlockEvent || pInfo->bNewFilesetEvent) { if (pInfo->bNextDurationBlockEvent || pInfo->bNewFilesetEvent) {
if (!bSkipped) { if (!bSkipped) {
int32_t code = createOneDataBlock(pBlock, true, &pInfo->nextDurationBlocks[pInfo->numNextDurationBlocks]); code = createOneDataBlock(pBlock, true, &pInfo->nextDurationBlocks[pInfo->numNextDurationBlocks]);
if (code) { if (code) {
terrno = code; terrno = code;
return NULL; *ppBlock = NULL;
return code;
} }
++pInfo->numNextDurationBlocks; ++pInfo->numNextDurationBlocks;
@ -5473,7 +5475,8 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
if (pInfo->bNewFilesetEvent) { if (pInfo->bNewFilesetEvent) {
pInfo->rtnNextDurationBlocks = true; pInfo->rtnNextDurationBlocks = true;
return NULL; *ppBlock = NULL;
return code;
} }
if (pInfo->bNextDurationBlockEvent) { if (pInfo->bNextDurationBlockEvent) {
@ -5488,11 +5491,13 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
pOperator->resultInfo.totalRows += pBlock->info.rows; pOperator->resultInfo.totalRows += pBlock->info.rows;
pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
*ppBlock = pBlock;
return pBlock;
return code;
} }
return NULL; *ppBlock = NULL;
return code;
} }
int32_t generateSortByTsPkInfo(SArray* colMatchInfo, int32_t order, SArray** ppSortArray) { int32_t generateSortByTsPkInfo(SArray* colMatchInfo, int32_t order, SArray** ppSortArray) {

18
source/libs/executor/src/sortoperator.c
View File

@ -337,10 +337,10 @@ static int32_t getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock,
return code; return code;
} }
SSDataBlock* loadNextDataBlock(void* param) { int32_t loadNextDataBlock(void* param, SSDataBlock** ppBlock) {
SOperatorInfo* pOperator = (SOperatorInfo*)param; SOperatorInfo* pOperator = (SOperatorInfo*)param;
SSDataBlock* pBlock = pOperator->fpSet.getNextFn(pOperator); *ppBlock = pOperator->fpSet.getNextFn(pOperator);
return pBlock; return TSDB_CODE_SUCCESS;
} }
// todo refactor: merged with fetch fp // todo refactor: merged with fetch fp
@ -609,30 +609,32 @@ typedef struct SGroupSortSourceParam {
SGroupSortOperatorInfo* grpSortOpInfo; SGroupSortOperatorInfo* grpSortOpInfo;
} SGroupSortSourceParam; } SGroupSortSourceParam;
SSDataBlock* fetchNextGroupSortDataBlock(void* param) { int32_t fetchNextGroupSortDataBlock(void* param, SSDataBlock** ppBlock) {
*ppBlock = NULL;
SGroupSortSourceParam* source = param; SGroupSortSourceParam* source = param;
SGroupSortOperatorInfo* grpSortOpInfo = source->grpSortOpInfo; SGroupSortOperatorInfo* grpSortOpInfo = source->grpSortOpInfo;
if (grpSortOpInfo->prefetchedSortInput) { if (grpSortOpInfo->prefetchedSortInput) {
SSDataBlock* block = grpSortOpInfo->prefetchedSortInput; SSDataBlock* block = grpSortOpInfo->prefetchedSortInput;
grpSortOpInfo->prefetchedSortInput = NULL; grpSortOpInfo->prefetchedSortInput = NULL;
return block; *ppBlock = block;
} else { } else {
SOperatorInfo* childOp = source->childOpInfo; SOperatorInfo* childOp = source->childOpInfo;
SSDataBlock* block = childOp->fpSet.getNextFn(childOp); SSDataBlock* block = childOp->fpSet.getNextFn(childOp);
if (block != NULL) { if (block != NULL) {
if (block->info.id.groupId == grpSortOpInfo->currGroupId) { if (block->info.id.groupId == grpSortOpInfo->currGroupId) {
grpSortOpInfo->childOpStatus = CHILD_OP_SAME_GROUP; grpSortOpInfo->childOpStatus = CHILD_OP_SAME_GROUP;
return block; *ppBlock = block;
} else { } else {
grpSortOpInfo->childOpStatus = CHILD_OP_NEW_GROUP; grpSortOpInfo->childOpStatus = CHILD_OP_NEW_GROUP;
grpSortOpInfo->prefetchedSortInput = block; grpSortOpInfo->prefetchedSortInput = block;
return NULL;
} }
} else { } else {
grpSortOpInfo->childOpStatus = CHILD_OP_FINISHED; grpSortOpInfo->childOpStatus = CHILD_OP_FINISHED;
return NULL;
} }
} }
return TSDB_CODE_SUCCESS;
} }
int32_t beginSortGroup(SOperatorInfo* pOperator) { int32_t beginSortGroup(SOperatorInfo* pOperator) {

2
source/libs/executor/src/sysscanoperator.c
View File

@ -1436,7 +1436,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
int64_t suid = mr.me.ctbEntry.suid; int64_t suid = mr.me.ctbEntry.suid;
code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid); code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", mr.me.name,
suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
pAPI->metaReaderFn.clearReader(&mr1); pAPI->metaReaderFn.clearReader(&mr1);
pAPI->metaReaderFn.clearReader(&mr); pAPI->metaReaderFn.clearReader(&mr);

21
source/libs/executor/src/tsort.c
View File

@ -619,7 +619,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32
for (int32_t i = 0; i < pParam->numOfSources; ++i) { for (int32_t i = 0; i < pParam->numOfSources; ++i) {
SSortSource* pSource = pParam->pSources[i]; SSortSource* pSource = pParam->pSources[i];
pSource->src.pBlock = pHandle->fetchfp(pSource->param); TAOS_CHECK_RETURN(pHandle->fetchfp(pSource->param, &pSource->src.pBlock));
// set current source is done // set current source is done
if (pSource->src.pBlock == NULL) { if (pSource->src.pBlock == NULL) {
@ -711,7 +711,7 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT
} }
} else { } else {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
pSource->src.pBlock = pHandle->fetchfp(((SSortSource*)pSource)->param); TAOS_CHECK_RETURN(pHandle->fetchfp(((SSortSource*)pSource)->param, &pSource->src.pBlock));
pSource->fetchUs += taosGetTimestampUs() - st; pSource->fetchUs += taosGetTimestampUs() - st;
pSource->fetchNum++; pSource->fetchNum++;
if (pSource->src.pBlock == NULL) { if (pSource->src.pBlock == NULL) {
@ -1350,7 +1350,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
} }
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath); taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
pMemFile->pTdFile = taosOpenCFile(pMemFile->memFilePath, "w+"); pMemFile->pTdFile = taosOpenCFile(pMemFile->memFilePath, "w+b");
if (pMemFile->pTdFile == NULL) { if (pMemFile->pTdFile == NULL) {
code = terrno = TAOS_SYSTEM_ERROR(errno); code = terrno = TAOS_SYSTEM_ERROR(errno);
} }
@ -2236,8 +2236,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
while (1) { while (1) {
bool bExtractedBlock = false; bool bExtractedBlock = false;
bool bSkipBlock = false; bool bSkipBlock = false;
SSDataBlock* pBlk = NULL;
SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param);
TAOS_CHECK_RETURN(pHandle->fetchfp(pSrc->param, &pBlk));
if (pBlk != NULL && pHandle->mergeLimit > 0) { if (pBlk != NULL && pHandle->mergeLimit > 0) {
SSDataBlock* p = NULL; SSDataBlock* p = NULL;
code = getRowsBlockWithinMergeLimit(pHandle, mTableNumRows, pBlk, &bExtractedBlock, &bSkipBlock, &p); code = getRowsBlockWithinMergeLimit(pHandle, mTableNumRows, pBlk, &bExtractedBlock, &bSkipBlock, &p);
@ -2390,7 +2391,8 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL);
while (1) { while (1) {
SSDataBlock* pBlock = pHandle->fetchfp(source->param); SSDataBlock* pBlock = NULL;
TAOS_CHECK_RETURN(pHandle->fetchfp(source->param, &pBlock));
if (pBlock == NULL) { if (pBlock == NULL) {
break; break;
} }
@ -2701,7 +2703,8 @@ static int32_t tsortOpenForPQSort(SSortHandle* pHandle) {
while (1) { while (1) {
// fetch data // fetch data
SSDataBlock* pBlock = pHandle->fetchfp(source->param); SSDataBlock* pBlock = NULL;
TAOS_CHECK_RETURN(pHandle->fetchfp(source->param, &pBlock));
if (NULL == pBlock) { if (NULL == pBlock) {
break; break;
} }
@ -2828,7 +2831,9 @@ static int32_t tsortSingleTableMergeNextTuple(SSortHandle* pHandle, STupleHandle
} }
SSortSource* source = *pSource; SSortSource* source = *pSource;
SSDataBlock* pBlock = pHandle->fetchfp(source->param); SSDataBlock* pBlock = NULL;
TAOS_CHECK_RETURN(pHandle->fetchfp(source->param, &pBlock));
if (!pBlock || pBlock->info.rows == 0) { if (!pBlock || pBlock->info.rows == 0) {
setCurrentSourceDone(source, pHandle); setCurrentSourceDone(source, pHandle);
pHandle->tupleHandle.pBlock = NULL; pHandle->tupleHandle.pBlock = NULL;

36
source/libs/monitor/src/monMain.c
View File

@ -21,11 +21,25 @@
#include "thttp.h" #include "thttp.h"
#include "ttime.h" #include "ttime.h"
#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count"
#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"
#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id"
#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep"
#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id"
#define VNODE_METRIC_TAG_NAME_USERNAME "username"
#define VNODE_METRIC_TAG_NAME_RESULT "result"
// #define VNODE_METRIC_TAG_VALUE_INSERT "insert"
// #define VNODE_METRIC_TAG_VALUE_DELETE "delete"
SMonitor tsMonitor = {0}; SMonitor tsMonitor = {0};
char *tsMonUri = "/report"; char *tsMonUri = "/report";
char *tsMonFwUri = "/general-metric"; char *tsMonFwUri = "/general-metric";
char *tsMonSlowLogUri = "/slow-sql-detail-batch"; char *tsMonSlowLogUri = "/slow-sql-detail-batch";
char *tsMonFwBasicUri = "/taosd-cluster-basic"; char *tsMonFwBasicUri = "/taosd-cluster-basic";
taos_counter_t *tsInsertCounter = NULL;
void monRecordLog(int64_t ts, ELogLevel level, const char *content) { void monRecordLog(int64_t ts, ELogLevel level, const char *content) {
(void)taosThreadMutexLock(&tsMonitor.lock); (void)taosThreadMutexLock(&tsMonitor.lock);
@ -117,6 +131,28 @@ int32_t monInit(const SMonCfg *pCfg) {
return 0; return 0;
} }
void monInitVnode() {
if (tsEnableMonitor && tsMonitorFqdn[0] != 0 && tsMonitorPort != 0 && tsInsertCounter == NULL) {
taos_counter_t *counter = NULL;
int32_t label_count = 7;
const char *sample_labels[] = {VNODE_METRIC_TAG_NAME_SQL_TYPE, VNODE_METRIC_TAG_NAME_CLUSTER_ID,
VNODE_METRIC_TAG_NAME_DNODE_ID, VNODE_METRIC_TAG_NAME_DNODE_EP,
VNODE_METRIC_TAG_NAME_VGROUP_ID, VNODE_METRIC_TAG_NAME_USERNAME,
VNODE_METRIC_TAG_NAME_RESULT};
counter = taos_counter_new(VNODE_METRIC_SQL_COUNT, "counter for insert sql", label_count, sample_labels);
uDebug("new metric:%p", counter);
if (taos_collector_registry_register_metric(counter) == 1) {
(void)taos_counter_destroy(counter);
uError("failed to register metric:%p", counter);
} else {
tsInsertCounter = counter;
uInfo("succeed to set inserted row metric:%p", tsInsertCounter);
}
} else {
uError("failed to set insert counter, already set");
}
}
void monCleanup() { void monCleanup() {
tsLogFp = NULL; tsLogFp = NULL;
taosArrayDestroy(tsMonitor.logs); taosArrayDestroy(tsMonitor.logs);

98
source/libs/stream/src/streamCheckStatus.c
View File

@ -36,7 +36,7 @@ static void setCheckDownstreamReqInfo(SStreamTaskCheckReq* pReq, int64_t reqId,
static void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault, static void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault,
int32_t* numOfNotRsp, SArray* pTimeoutList, SArray* pNotReadyList, const char* id); int32_t* numOfNotRsp, SArray* pTimeoutList, SArray* pNotReadyList, const char* id);
static int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId); static int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId);
static void findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId, SDownstreamStatusInfo** pStatusInfo); static void findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId, SDownstreamStatusInfo** pStatusInfo);
int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage,
int64_t* oldStage) { int64_t* oldStage) {
@ -107,11 +107,12 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) {
streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pDispatch->taskId, pDispatch->nodeId, idstr); streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pDispatch->taskId, pDispatch->nodeId, idstr);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64 stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64
" window:%" PRId64 "-%" PRId64 " reqId:0x%" PRIx64, " window:%" PRId64 "-%" PRId64 " QID:0x%" PRIx64,
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer,
pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId);
(void) streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet); (void)streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId,
&pTask->outputInfo.fixedDispatcher.epSet);
} else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
streamTaskStartMonitorCheckRsp(pTask); streamTaskStartMonitorCheckRsp(pTask);
@ -132,9 +133,9 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) {
streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pVgInfo->taskId, pVgInfo->vgId, idstr); streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pVgInfo->taskId, pVgInfo->vgId, idstr);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 stDebug("s-task:%s (vgId:%d) stage:%" PRId64
" check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64, " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, QID:0x%" PRIx64,
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId); idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId);
(void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); (void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
} }
} else { // for sink task, set it ready directly. } else { // for sink task, set it ready directly.
stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId); stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId);
@ -164,20 +165,20 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS
pRsp->status = TASK_DOWNSTREAM_NOT_LEADER; pRsp->status = TASK_DOWNSTREAM_NOT_LEADER;
} else { } else {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, taskId, &pTask);
if (pTask != NULL) { if (pTask != NULL) {
pRsp->status = pRsp->status =
streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64 stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(QID:0x%" PRIx64
") task:0x%x (vgId:%d), check_status:%d", ") task:0x%x (vgId:%d), check_status:%d",
pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId,
pRsp->status); pRsp->status);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} else { } else {
pRsp->status = TASK_DOWNSTREAM_NOT_READY; pRsp->status = TASK_DOWNSTREAM_NOT_READY;
stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(QID:0x%" PRIx64
") from task:0x%x (vgId:%d), rsp check_status %d", ") from task:0x%x (vgId:%d), rsp check_status %d",
pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status);
} }
@ -261,7 +262,7 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tEncoderInit(&encoder, (uint8_t*)abuf, len); tEncoderInit(&encoder, (uint8_t*)abuf, len);
(void) tEncodeStreamTaskCheckRsp(&encoder, pRsp); (void)tEncodeStreamTaskCheckRsp(&encoder, pRsp);
tEncoderClear(&encoder); tEncoderClear(&encoder);
SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo};
@ -300,7 +301,8 @@ void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) {
if (pInfo->checkRspTmr == NULL) { if (pInfo->checkRspTmr == NULL) {
pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer); pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer);
} else { } else {
streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor"); streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId,
"check-status-monitor");
} }
streamMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
@ -319,7 +321,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
pInfo->pList = NULL; pInfo->pList = NULL;
if (pInfo->checkRspTmr != NULL) { if (pInfo->checkRspTmr != NULL) {
(void) taosTmrStop(pInfo->checkRspTmr); (void)taosTmrStop(pInfo->checkRspTmr);
pInfo->checkRspTmr = NULL; pInfo->checkRspTmr = NULL;
} }
@ -329,11 +331,11 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void processDownstreamReadyRsp(SStreamTask* pTask) { void processDownstreamReadyRsp(SStreamTask* pTask) {
EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
(void) streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); (void)streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL);
int64_t checkTs = pTask->execInfo.checkTs; int64_t checkTs = pTask->execInfo.checkTs;
int64_t readyTs = pTask->execInfo.readyTs; int64_t readyTs = pTask->execInfo.readyTs;
(void) streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true); (void)streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true);
if (pTask->status.taskStatus == TASK_STATUS__HALT) { if (pTask->status.taskStatus == TASK_STATUS__HALT) {
if (!HAS_RELATED_FILLHISTORY_TASK(pTask) || (pTask->info.fillHistory != 0)) { if (!HAS_RELATED_FILLHISTORY_TASK(pTask) || (pTask->info.fillHistory != 0)) {
@ -355,7 +357,7 @@ void processDownstreamReadyRsp(SStreamTask* pTask) {
// todo: let's retry // todo: let's retry
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr); stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr);
(void) streamLaunchFillHistoryTask(pTask); (void)streamLaunchFillHistoryTask(pTask);
} }
} }
@ -379,7 +381,7 @@ void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) {
if (!existed) { if (!existed) {
SDownstreamTaskEpset t = {.nodeId = nodeId}; SDownstreamTaskEpset t = {.nodeId = nodeId};
void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t);
if (p == NULL) { if (p == NULL) {
// todo let's retry // todo let's retry
} }
@ -430,7 +432,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
findCheckRspStatus(pInfo, taskId, &p); findCheckRspStatus(pInfo, taskId, &p);
if (p != NULL) { if (p != NULL) {
if (reqId != p->reqId) { if (reqId != p->reqId) {
stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 stError("s-task:%s QID:0x%" PRIx64 " expected:0x%" PRIx64
" expired check-rsp recv from downstream task:0x%x, discarded", " expired check-rsp recv from downstream task:0x%x, discarded",
id, reqId, p->reqId, taskId); id, reqId, p->reqId, taskId);
streamMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
@ -452,7 +454,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
} }
streamMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId, stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, QID:%" PRIx64 " discarded", id, taskId,
reqId); reqId);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -539,10 +541,10 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher; STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher;
setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId); setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id, stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) QID:0x%" PRIx64, id,
pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId);
(void) streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); (void)streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet);
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) { } else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos; SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos;
int32_t numOfVgs = taosArrayGetSize(vgInfo); int32_t numOfVgs = taosArrayGetSize(vgInfo);
@ -557,9 +559,9 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
setCheckDownstreamReqInfo(&req, p->reqId, pVgInfo->taskId, pVgInfo->vgId); setCheckDownstreamReqInfo(&req, p->reqId, pVgInfo->taskId, pVgInfo->vgId);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 stDebug("s-task:%s (vgId:%d) stage:%" PRId64
" re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64, " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d QID:0x%" PRIx64,
id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId); id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId);
(void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); (void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
break; break;
} }
} }
@ -580,15 +582,15 @@ void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, i
stDebug("s-task:%s recv status:NEW_STAGE/NOT_LEADER from downstream, task:0x%x, quit from check downstream", id, stDebug("s-task:%s recv status:NEW_STAGE/NOT_LEADER from downstream, task:0x%x, quit from check downstream", id,
p->taskId); p->taskId);
(*numOfFault) += 1; (*numOfFault) += 1;
} else { // TASK_DOWNSTREAM_NOT_READY } else { // TASK_DOWNSTREAM_NOT_READY
if (p->rspTs == 0) { // not response yet if (p->rspTs == 0) { // not response yet
if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec.
(void) taosArrayPush(pTimeoutList, &p->taskId); (void)taosArrayPush(pTimeoutList, &p->taskId);
} else { // el < CHECK_NOT_RSP_DURATION } else { // el < CHECK_NOT_RSP_DURATION
(*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp (*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp
} }
} else { } else {
(void) taosArrayPush(pNotReadyList, &p->taskId); (void)taosArrayPush(pNotReadyList, &p->taskId);
} }
} }
} }
@ -613,14 +615,13 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) {
continue; continue;
} }
int32_t taskId = *px; int32_t taskId = *px;
SDownstreamStatusInfo* p = NULL; SDownstreamStatusInfo* p = NULL;
findCheckRspStatus(pInfo, taskId, &p); findCheckRspStatus(pInfo, taskId, &p);
if (p != NULL) { if (p != NULL) {
if (p->status != -1 || p->rspTs != 0) { if (p->status != -1 || p->rspTs != 0) {
stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%"PRId64, pTask->id.idStr, i, p->status, stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%" PRId64, pTask->id.idStr, i,
p->rspTs); p->status, p->rspTs);
continue; continue;
} }
@ -692,24 +693,24 @@ int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64
// this function is executed in timer thread // this function is executed in timer thread
void rspMonitorFn(void* param, void* tmrId) { void rspMonitorFn(void* param, void* tmrId) {
SStreamTask* pTask = param; SStreamTask* pTask = param;
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
STaskCheckInfo* pInfo = &pTask->taskCheckInfo; STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
int32_t vgId = pTask->pMeta->vgId; int32_t vgId = pTask->pMeta->vgId;
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
int64_t timeoutDuration = now - pInfo->timeoutStartTs; int64_t timeoutDuration = now - pInfo->timeoutStartTs;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t numOfReady = 0; int32_t numOfReady = 0;
int32_t numOfFault = 0; int32_t numOfFault = 0;
int32_t numOfNotRsp = 0; int32_t numOfNotRsp = 0;
int32_t numOfNotReady = 0; int32_t numOfNotReady = 0;
int32_t numOfTimeout = 0; int32_t numOfTimeout = 0;
int32_t total = taosArrayGetSize(pInfo->pList); int32_t total = taosArrayGetSize(pInfo->pList);
stDebug("s-task:%s start to do check-downstream-rsp check in tmr", id); stDebug("s-task:%s start to do check-downstream-rsp check in tmr", id);
streamMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState state = streamTaskGetStatus(pTask); SStreamTaskState state = streamTaskGetStatus(pTask);
streamMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if (state.state == TASK_STATUS__STOP) { if (state.state == TASK_STATUS__STOP) {
@ -721,7 +722,7 @@ void rspMonitorFn(void* param, void* tmrId) {
// not record the failed of the current task if try to close current vnode // not record the failed of the current task if try to close current vnode
// otherwise, the put of message operation may incur invalid read of message queue. // otherwise, the put of message operation may incur invalid read of message queue.
if (!pMeta->closeFlag) { if (!pMeta->closeFlag) {
(void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); (void)addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -740,8 +741,8 @@ void rspMonitorFn(void* param, void* tmrId) {
streamMutexLock(&pInfo->checkInfoLock); streamMutexLock(&pInfo->checkInfoLock);
if (pInfo->notReadyTasks == 0) { if (pInfo->notReadyTasks == 0) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, state.name, stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, state.name, vgId,
vgId, ref); ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
streamMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
@ -803,7 +804,7 @@ void rspMonitorFn(void* param, void* tmrId) {
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
streamMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
(void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); (void)addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
taosArrayDestroy(pNotReadyList); taosArrayDestroy(pNotReadyList);
@ -819,7 +820,8 @@ void rspMonitorFn(void* param, void* tmrId) {
handleTimeoutDownstreamTasks(pTask, pTimeoutList); handleTimeoutDownstreamTasks(pTask, pTimeoutList);
} }
streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor"); streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId,
"check-status-monitor");
streamMutexUnlock(&pInfo->checkInfoLock); streamMutexUnlock(&pInfo->checkInfoLock);
stDebug( stDebug(

51
source/libs/stream/src/streamDispatch.c
View File

@ -118,7 +118,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, len); tEncoderInit(&encoder, abuf, len);
(void) tEncodeStreamRetrieveReq(&encoder, req); (void)tEncodeStreamRetrieveReq(&encoder, req);
tEncoderClear(&encoder); tEncoderClear(&encoder);
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};
@ -130,7 +130,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
return code; return code;
} }
stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), QID:0x%" PRIx64, pTask->id.idStr,
pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId); pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId);
} }
@ -254,7 +254,7 @@ static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStre
int32_t type = pTask->outputInfo.type; int32_t type = pTask->outputInfo.type;
int32_t num = streamTaskGetNumOfDownstream(pTask); int32_t num = streamTaskGetNumOfDownstream(pTask);
if(type != TASK_OUTPUT__SHUFFLE_DISPATCH && type != TASK_OUTPUT__FIXED_DISPATCH) { if (type != TASK_OUTPUT__SHUFFLE_DISPATCH && type != TASK_OUTPUT__FIXED_DISPATCH) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
stError("s-task:%s invalid dispatch type:%d not dispatch data", pTask->id.idStr, type); stError("s-task:%s invalid dispatch type:%d not dispatch data", pTask->id.idStr, type);
return NULL; return NULL;
@ -283,7 +283,7 @@ static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStre
return NULL; return NULL;
} }
} }
} else { // shuffle dispatch } else { // shuffle dispatch
int32_t numOfBlocks = taosArrayGetSize(pData->blocks); int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId;
@ -470,7 +470,7 @@ static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t
streamMutexLock(&pMsgInfo->lock); streamMutexLock(&pMsgInfo->lock);
} }
(void) taosArrayPush(pMsgInfo->pSendInfo, &entry); (void)taosArrayPush(pMsgInfo->pSendInfo, &entry);
if (lock) { if (lock) {
streamMutexUnlock(&pMsgInfo->lock); streamMutexUnlock(&pMsgInfo->lock);
@ -591,7 +591,7 @@ static void doMonitorDispatchData(void* param, void* tmrId) {
SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet; SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet;
int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId;
int32_t s = taosArrayGetSize(pTask->msgInfo.pSendInfo); int32_t s = taosArrayGetSize(pTask->msgInfo.pSendInfo);
SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, 0); SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, 0);
if (pEntry != NULL) { if (pEntry != NULL) {
setResendInfo(pEntry, now); setResendInfo(pEntry, now);
@ -617,7 +617,8 @@ static void doMonitorDispatchData(void* param, void* tmrId) {
void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) {
int32_t vgId = pTask->pMeta->vgId; int32_t vgId = pTask->pMeta->vgId;
if (pTask->msgInfo.pRetryTmr != NULL) { if (pTask->msgInfo.pRetryTmr != NULL) {
streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, "dispatch-monitor-tmr"); streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId,
"dispatch-monitor-tmr");
} else { } else {
pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer); pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer);
} }
@ -651,7 +652,8 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
} }
} }
} else { } else {
(void) buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); (void)buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId,
pDataBlock->info.parTbName);
} }
snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db,
@ -666,7 +668,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
// failed to put into name buffer, no need to do anything // failed to put into name buffer, no need to do anything
if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) {
(void) tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); (void)tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName));
} }
} }
@ -848,7 +850,8 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
} }
if (++pTmrInfo->activeCounter < 50) { if (++pTmrInfo->activeCounter < 50) {
streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
"chkpt-ready-monitor");
return; return;
} }
@ -907,7 +910,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
continue; continue;
} }
(void) taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); (void)taosArrayPush(pNotRspList, &pInfo->upstreamTaskId);
stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId,
pTask->info.taskLevel, pInfo->upstreamTaskId); pTask->info.taskLevel, pInfo->upstreamTaskId);
} }
@ -931,8 +934,8 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
if (*pTaskId == pReadyInfo->upstreamTaskId) { // send msg again if (*pTaskId == pReadyInfo->upstreamTaskId) { // send msg again
SRpcMsg msg = {0}; SRpcMsg msg = {0};
int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId, int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId,
checkpointId, &msg); pReadyInfo->childId, checkpointId, &msg);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
@ -948,7 +951,8 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
} }
} }
streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
"chkpt-ready-monitor");
streamMutexUnlock(&pActiveInfo->lock); streamMutexUnlock(&pActiveInfo->lock);
} else { } else {
int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask);
@ -1015,7 +1019,8 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
if (pTmrInfo->tmrHandle == NULL) { if (pTmrInfo->tmrHandle == NULL) {
pTmrInfo->tmrHandle = taosTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer); pTmrInfo->tmrHandle = taosTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer);
} else { } else {
streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor"); streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
"chkpt-ready-monitor");
} }
// mark the timer monitor checkpointId // mark the timer monitor checkpointId
@ -1087,8 +1092,8 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch
payloadLen += sizeof(SRetrieveTableRsp); payloadLen += sizeof(SRetrieveTableRsp);
(void) taosArrayPush(pReq->dataLen, &payloadLen); (void)taosArrayPush(pReq->dataLen, &payloadLen);
(void) taosArrayPush(pReq->data, &buf); (void)taosArrayPush(pReq->data, &buf);
pReq->totalLen += dataStrLen; pReq->totalLen += dataStrLen;
return 0; return 0;
@ -1169,7 +1174,7 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
tEncoderInit(&encoder, (uint8_t*)abuf, len); tEncoderInit(&encoder, (uint8_t*)abuf, len);
(void) tEncodeStreamCheckpointSourceRsp(&encoder, &rsp); (void)tEncodeStreamCheckpointSourceRsp(&encoder, &rsp);
tEncoderClear(&encoder); tEncoderClear(&encoder);
initRpcMsg(pMsg, 0, pBuf, sizeof(SMsgHead) + len); initRpcMsg(pMsg, 0, pBuf, sizeof(SMsgHead) + len);
@ -1185,7 +1190,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
.recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId}; .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId};
// todo retry until it success // todo retry until it success
(void) streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS); (void)streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS);
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
streamMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
@ -1208,7 +1213,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId); pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId);
} }
} else { } else {
(void) taosArrayPush(pActiveInfo->pReadyMsgList, &info); (void)taosArrayPush(pActiveInfo->pReadyMsgList, &info);
stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1); stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1);
} }
@ -1217,7 +1222,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
} }
void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId,
int32_t childId, SEpSet* pEpset, int64_t checkpointId) { int32_t childId, SEpSet* pEpset, int64_t checkpointId) {
pReadyInfo->upstreamTaskId = upstreamTaskId; pReadyInfo->upstreamTaskId = upstreamTaskId;
pReadyInfo->upstreamNodeEpset = *pEpset; pReadyInfo->upstreamNodeEpset = *pEpset;
pReadyInfo->upstreamNodeId = upstreamNodeId; pReadyInfo->upstreamNodeId = upstreamNodeId;
@ -1246,7 +1251,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
streamMutexLock(&pActiveInfo->lock); streamMutexLock(&pActiveInfo->lock);
(void) taosArrayPush(pActiveInfo->pReadyMsgList, &info); (void)taosArrayPush(pActiveInfo->pReadyMsgList, &info);
int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList); int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList);
int32_t total = streamTaskGetNumOfUpstream(pTask); int32_t total = streamTaskGetNumOfUpstream(pTask);
@ -1282,7 +1287,7 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId
stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData);
int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs;
bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER);
clearBufferedDispatchMsg(pTask); clearBufferedDispatchMsg(pTask);

81
source/libs/stream/src/streamExec.c
View File

@ -18,10 +18,10 @@
// maximum allowed processed block batches. One block may include several submit blocks // maximum allowed processed block batches. One block may include several submit blocks
#define MAX_STREAM_EXEC_BATCH_NUM 32 #define MAX_STREAM_EXEC_BATCH_NUM 32
#define STREAM_RESULT_DUMP_THRESHOLD 300 #define STREAM_RESULT_DUMP_THRESHOLD 300
#define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1) // 1MiB result data #define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1) // 1MiB result data
#define STREAM_SCAN_HISTORY_TIMESLICE 1000 // 1000 ms #define STREAM_SCAN_HISTORY_TIMESLICE 1000 // 1000 ms
#define MIN_INVOKE_INTERVAL 50 // 50ms #define MIN_INVOKE_INTERVAL 50 // 50ms
#define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec #define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec
static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); static int32_t streamTransferStateDoPrepare(SStreamTask* pTask);
static void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); static void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks);
@ -54,7 +54,7 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl
// not handle error, if dispatch failed, try next time. // not handle error, if dispatch failed, try next time.
// checkpoint trigger will be checked // checkpoint trigger will be checked
(void) streamDispatchStreamBlock(pTask); (void)streamDispatchStreamBlock(pTask);
} }
return code; return code;
@ -78,7 +78,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray*
} }
stDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, stDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks,
SIZE_IN_MiB(size)); SIZE_IN_MiB(size));
code = doOutputResultBlockImpl(pTask, pStreamBlocks); code = doOutputResultBlockImpl(pTask, pStreamBlocks);
if (code != TSDB_CODE_SUCCESS) { // back pressure and record position if (code != TSDB_CODE_SUCCESS) { // back pressure and record position
@ -99,7 +99,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
*totalSize = 0; *totalSize = 0;
int32_t size = 0; int32_t size = 0;
int32_t numOfBlocks= 0; int32_t numOfBlocks = 0;
SArray* pRes = NULL; SArray* pRes = NULL;
while (1) { while (1) {
@ -129,7 +129,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem;
ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
(void) assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); (void)assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0));
block.info.type = STREAM_PULL_OVER; block.info.type = STREAM_PULL_OVER;
block.info.childId = pTask->info.selfChildId; block.info.childId = pTask->info.selfChildId;
@ -140,8 +140,8 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
stError("s-task:%s failed to add retrieve block", pTask->id.idStr); stError("s-task:%s failed to add retrieve block", pTask->id.idStr);
} }
stDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64 " dump results", pTask->id.idStr, stDebug("s-task:%s(child %d) retrieve process completed, QID:0x%" PRIx64 " dump results", pTask->id.idStr,
pTask->info.selfChildId, pRetrieveBlock->reqId); pTask->info.selfChildId, pRetrieveBlock->reqId);
} }
break; break;
@ -174,7 +174,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
} }
stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr,
pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size)); pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size));
// current output should be dispatched to down stream nodes // current output should be dispatched to down stream nodes
if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) {
@ -251,7 +251,7 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t*
} }
SSDataBlock block = {0}; SSDataBlock block = {0};
(void) assignOneDataBlock(&block, output); (void)assignOneDataBlock(&block, output);
block.info.childId = pTask->info.selfChildId; block.info.childId = pTask->info.selfChildId;
void* p = taosArrayPush(pRes, &block); void* p = taosArrayPush(pRes, &block);
@ -259,7 +259,8 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t*
stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr); stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr);
} }
(*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); (*pSize) +=
blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
numOfBlocks += 1; numOfBlocks += 1;
if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || (*pSize) >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || (*pSize) >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) {
@ -283,7 +284,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
if (!pTask->hTaskInfo.operatorOpen) { if (!pTask->hTaskInfo.operatorOpen) {
(void) qSetStreamOpOpen(exec); (void)qSetStreamOpOpen(exec);
pTask->hTaskInfo.operatorOpen = true; pTask->hTaskInfo.operatorOpen = true;
} }
@ -315,13 +316,13 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
int32_t size = 0; int32_t size = 0;
streamScanHistoryDataImpl(pTask, pRes, &size, &finished); streamScanHistoryDataImpl(pTask, pRes, &size, &finished);
if(streamTaskShouldStop(pTask)) { if (streamTaskShouldStop(pTask)) {
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0); return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0);
} }
// dispatch the generated results, todo fix error // dispatch the generated results, todo fix error
(void) handleScanhistoryResultBlocks(pTask, pRes, size); (void)handleScanhistoryResultBlocks(pTask, pRes, size);
if (finished) { if (finished) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0); return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0);
@ -346,11 +347,11 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
stError( stError(
"s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related " "s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related "
"fill-history task", "fill-history task",
id, (int32_t) pTask->streamTaskId.taskId); id, (int32_t)pTask->streamTaskId.taskId);
// 1. free it and remove fill-history task from disk meta-store // 1. free it and remove fill-history task from disk meta-store
// todo: this function should never be failed. // todo: this function should never be failed.
(void) streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); (void)streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
// 2. save to disk // 2. save to disk
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
@ -367,7 +368,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
id, streamTaskGetStatus(pTask).name, el, pStreamTask->id.idStr); id, streamTaskGetStatus(pTask).name, el, pStreamTask->id.idStr);
} }
ETaskStatus status = streamTaskGetStatus(pStreamTask).state; ETaskStatus status = streamTaskGetStatus(pStreamTask).state;
STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; STimeWindow* pTimeWindow = &pStreamTask->dataRange.window;
// It must be halted for a source stream task, since when the related scan-history-data task start scan the history // It must be halted for a source stream task, since when the related scan-history-data task start scan the history
@ -408,14 +409,14 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN, pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN,
pTimeWindow->ekey, p, pStreamTask->status.schedStatus); pTimeWindow->ekey, p, pStreamTask->status.schedStatus);
(void) streamTaskResetTimewindowFilter(pStreamTask); (void)streamTaskResetTimewindowFilter(pStreamTask);
} else { } else {
stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr); stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr);
} }
// NOTE: transfer the ownership of executor state before handle the checkpoint block during stream exec // NOTE: transfer the ownership of executor state before handle the checkpoint block during stream exec
// 2. send msg to mnode to launch a checkpoint to keep the state for current stream // 2. send msg to mnode to launch a checkpoint to keep the state for current stream
(void) streamTaskSendCheckpointReq(pStreamTask); (void)streamTaskSendCheckpointReq(pStreamTask);
// 3. assign the status to the value that will be kept in disk // 3. assign the status to the value that will be kept in disk
pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state; pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state;
@ -429,12 +430,12 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
static int32_t haltCallback(SStreamTask* pTask, void* param) { static int32_t haltCallback(SStreamTask* pTask, void* param) {
streamTaskOpenAllUpstreamInput(pTask); streamTaskOpenAllUpstreamInput(pTask);
(void) streamTaskSendCheckpointReq(pTask); (void)streamTaskSendCheckpointReq(pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t streamTransferStatePrepare(SStreamTask* pTask) { int32_t streamTransferStatePrepare(SStreamTask* pTask) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
ASSERT(pTask->status.appendTranstateBlock == 1); ASSERT(pTask->status.appendTranstateBlock == 1);
@ -466,7 +467,7 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) {
// set input // set input
static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) { static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) {
void* pExecutor = pTask->exec.pExecutor; void* pExecutor = pTask->exec.pExecutor;
int32_t code = 0; int32_t code = 0;
const SStreamQueueItem* pItem = pInput; const SStreamQueueItem* pItem = pInput;
@ -479,7 +480,7 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int
const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput;
code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit,
pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver);
ASSERT((*pVer) <= pSubmit->submit.ver); ASSERT((*pVer) <= pSubmit->submit.ver);
(*pVer) = pSubmit->submit.ver; (*pVer) = pSubmit->submit.ver;
@ -497,7 +498,7 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int
SArray* pBlockList = pMerged->submits; SArray* pBlockList = pMerged->submits;
int32_t numOfBlocks = taosArrayGetSize(pBlockList); int32_t numOfBlocks = taosArrayGetSize(pBlockList);
stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks, stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks,
pMerged->ver); pMerged->ver);
code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT);
ASSERT((*pVer) <= pMerged->ver); ASSERT((*pVer) <= pMerged->ver);
(*pVer) = pMerged->ver; (*pVer) = pMerged->ver;
@ -549,7 +550,7 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock)
pBlock->srcVgId = pTask->pMeta->vgId; pBlock->srcVgId = pTask->pMeta->vgId;
code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
if (code == 0) { if (code == 0) {
(void) streamDispatchStreamBlock(pTask); (void)streamDispatchStreamBlock(pTask);
} else { // todo put into queue failed, retry } else { // todo put into queue failed, retry
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
} }
@ -568,7 +569,7 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock)
} }
} }
//static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } // static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; }
static void setLastExecTs(SStreamTask* pTask, int64_t ts) { pTask->status.lastExecTs = ts; } static void setLastExecTs(SStreamTask* pTask, int64_t ts) { pTask->status.lastExecTs = ts; }
static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, int32_t num) { static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, int32_t num) {
@ -581,7 +582,7 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i
stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type)); stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type));
int32_t code = doSetStreamInputBlock(pTask, pBlock, &ver, id); int32_t code = doSetStreamInputBlock(pTask, pBlock, &ver, id);
if(code) { if (code) {
stError("s-task:%s failed to set input block, not exec for these blocks", id); stError("s-task:%s failed to set input block, not exec for these blocks", id);
return; return;
} }
@ -609,7 +610,7 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i
if (ver != pInfo->processedVer) { if (ver != pInfo->processedVer) {
stDebug("s-task:%s update processedVer(unsaved) from %" PRId64 " to %" PRId64 " nextProcessVer:%" PRId64 stDebug("s-task:%s update processedVer(unsaved) from %" PRId64 " to %" PRId64 " nextProcessVer:%" PRId64
" ckpt:%" PRId64, " ckpt:%" PRId64,
id, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer); id, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer);
pInfo->processedVer = ver; pInfo->processedVer = ver;
} }
@ -625,10 +626,10 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB
STaskId* pHTaskId = &pTask->hTaskInfo.id; STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = NULL; SStreamTask* pHTask = NULL;
int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (code == TSDB_CODE_SUCCESS) { // ignore the error code. if (code == TSDB_CODE_SUCCESS) { // ignore the error code.
(void) streamTaskReleaseState(pHTask); (void)streamTaskReleaseState(pHTask);
(void) streamTaskReloadState(pTask); (void)streamTaskReloadState(pTask);
stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr, stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr,
streamTaskGetStatus(pHTask).name); streamTaskGetStatus(pHTask).name);
// todo execute qExecTask to fetch the reload-generated result, if this is stream is for session window query. // todo execute qExecTask to fetch the reload-generated result, if this is stream is for session window query.
@ -707,7 +708,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
// dispatch checkpoint msg to all downstream tasks // dispatch checkpoint msg to all downstream tasks
int32_t type = pInput->type; int32_t type = pInput->type;
if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
(void) streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); (void)streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput);
continue; continue;
} }
@ -744,14 +745,14 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
if (type != STREAM_INPUT__CHECKPOINT) { if (type != STREAM_INPUT__CHECKPOINT) {
doStreamTaskExecImpl(pTask, pInput, numOfBlocks); doStreamTaskExecImpl(pTask, pInput, numOfBlocks);
streamFreeQitem(pInput); streamFreeQitem(pInput);
} else { // todo other thread may change the status } else { // todo other thread may change the status
// do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed.
streamMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state == TASK_STATUS__CK) { if (pState.state == TASK_STATUS__CK) {
stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name);
(void) streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue (void)streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue
} else { // todo refactor } else { // todo refactor
int32_t code = 0; int32_t code = 0;
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
code = streamTaskSendCheckpointSourceRsp(pTask); code = streamTaskSendCheckpointSourceRsp(pTask);
@ -804,7 +805,7 @@ void streamResumeTask(SStreamTask* pTask) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
while (1) { while (1) {
(void) doStreamExecTask(pTask); (void)doStreamExecTask(pTask);
// check if continue // check if continue
streamMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);

2
source/libs/stream/src/streamTask.c
View File

@ -1092,7 +1092,7 @@ static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq*
} }
// enqueue // enqueue
stDebug("s-task:%s (vgId:%d level:%d) recv retrieve req from task:0x%x(vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, stDebug("s-task:%s (vgId:%d level:%d) recv retrieve req from task:0x%x(vgId:%d), QID:0x%" PRIx64, pTask->id.idStr,
pTask->pMeta->vgId, pTask->info.taskLevel, pReq->srcTaskId, pReq->srcNodeId, pReq->reqId); pTask->pMeta->vgId, pTask->info.taskLevel, pReq->srcTaskId, pReq->srcNodeId, pReq->reqId);
pData->type = STREAM_INPUT__DATA_RETRIEVE; pData->type = STREAM_INPUT__DATA_RETRIEVE;

4
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -57,14 +57,12 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
} }
if (ths->state == TAOS_SYNC_STATE_LEADER || ths->state == TAOS_SYNC_STATE_ASSIGNED_LEADER) { if (ths->state == TAOS_SYNC_STATE_LEADER || ths->state == TAOS_SYNC_STATE_ASSIGNED_LEADER) {
if (pMsg->term > raftStoreGetTerm(ths)) { if (pMsg->term != raftStoreGetTerm(ths)) {
syncLogRecvAppendEntriesReply(ths, pMsg, "error term"); syncLogRecvAppendEntriesReply(ths, pMsg, "error term");
syncNodeStepDown(ths, pMsg->term); syncNodeStepDown(ths, pMsg->term);
return TSDB_CODE_SYN_WRONG_TERM; return TSDB_CODE_SYN_WRONG_TERM;
} }
ASSERT(pMsg->term == raftStoreGetTerm(ths));
sTrace("vgId:%d, received append entries reply. srcId:0x%016" PRIx64 ", term:%" PRId64 ", matchIndex:%" PRId64 "", sTrace("vgId:%d, received append entries reply. srcId:0x%016" PRIx64 ", term:%" PRId64 ", matchIndex:%" PRId64 "",
pMsg->vgId, pMsg->srcId.addr, pMsg->term, pMsg->matchIndex); pMsg->vgId, pMsg->srcId.addr, pMsg->term, pMsg->matchIndex);

5
source/libs/sync/src/syncCommit.c
View File

@ -57,7 +57,10 @@ int32_t syncNodeDynamicQuorum(const SSyncNode* pSyncNode) { return pSyncNode->qu
bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index) { bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index) {
int count = 0; int count = 0;
SSyncIndexMgr* pMatches = pNode->pMatchIndex; SSyncIndexMgr* pMatches = pNode->pMatchIndex;
ASSERT(pNode->replicaNum == pMatches->replicaNum); if (pNode->replicaNum != pMatches->replicaNum) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return false;
};
for (int i = 0; i < pNode->totalReplicaNum; i++) { for (int i = 0; i < pNode->totalReplicaNum; i++) {
if(pNode->raftCfg.cfg.nodeInfo[i].nodeRole == TAOS_SYNC_ROLE_VOTER){ if(pNode->raftCfg.cfg.nodeInfo[i].nodeRole == TAOS_SYNC_ROLE_VOTER){

2
source/libs/sync/src/syncElection.c
View File

@ -119,7 +119,7 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) {
} }
ret = syncNodeRequestVotePeers(pSyncNode); ret = syncNodeRequestVotePeers(pSyncNode);
ASSERT(ret == 0); if (ret != 0) return ret;
syncNodeResetElectTimer(pSyncNode); syncNodeResetElectTimer(pSyncNode);
return ret; return ret;

36
source/libs/sync/src/syncMain.c
View File

@ -988,9 +988,18 @@ static int32_t syncHbTimerStop(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) {
int32_t syncNodeLogStoreRestoreOnNeed(SSyncNode* pNode) { int32_t syncNodeLogStoreRestoreOnNeed(SSyncNode* pNode) {
int32_t code = 0; int32_t code = 0;
ASSERTS(pNode->pLogStore != NULL, "log store not created"); if (pNode->pLogStore == NULL) {
ASSERTS(pNode->pFsm != NULL, "pFsm not registered"); sError("vgId:%d, log store not created", pNode->vgId);
ASSERTS(pNode->pFsm->FpGetSnapshotInfo != NULL, "FpGetSnapshotInfo not registered"); return TSDB_CODE_SYN_INTERNAL_ERROR;
}
if (pNode->pFsm == NULL) {
sError("vgId:%d, pFsm not registered", pNode->vgId);
return TSDB_CODE_SYN_INTERNAL_ERROR;
}
if (pNode->pFsm->FpGetSnapshotInfo == NULL) {
sError("vgId:%d, FpGetSnapshotInfo not registered", pNode->vgId);
return TSDB_CODE_SYN_INTERNAL_ERROR;
}
SSnapshot snapshot = {0}; SSnapshot snapshot = {0};
// TODO check return value // TODO check return value
(void)pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot); (void)pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot);
@ -1384,8 +1393,14 @@ void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) {
int32_t syncNodeRestore(SSyncNode* pSyncNode) { int32_t syncNodeRestore(SSyncNode* pSyncNode) {
int32_t code = 0; int32_t code = 0;
ASSERTS(pSyncNode->pLogStore != NULL, "log store not created"); if (pSyncNode->pLogStore == NULL) {
ASSERTS(pSyncNode->pLogBuf != NULL, "ring log buffer not created"); sError("vgId:%d, log store not created", pSyncNode->vgId);
return TSDB_CODE_SYN_INTERNAL_ERROR;
}
if (pSyncNode->pLogBuf == NULL) {
sError("vgId:%d, ring log buffer not created", pSyncNode->vgId);
return TSDB_CODE_SYN_INTERNAL_ERROR;
}
(void)taosThreadMutexLock(&pSyncNode->pLogBuf->mutex); (void)taosThreadMutexLock(&pSyncNode->pLogBuf->mutex);
SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
@ -1400,7 +1415,7 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) {
TAOS_RETURN(code); TAOS_RETURN(code);
} }
ASSERT(endIndex == lastVer + 1); if (endIndex != lastVer + 1) return TSDB_CODE_SYN_INTERNAL_ERROR;
pSyncNode->commitIndex = TMAX(pSyncNode->commitIndex, commitIndex); pSyncNode->commitIndex = TMAX(pSyncNode->commitIndex, commitIndex);
sInfo("vgId:%d, restore sync until commitIndex:%" PRId64, pSyncNode->vgId, pSyncNode->commitIndex); sInfo("vgId:%d, restore sync until commitIndex:%" PRId64, pSyncNode->vgId, pSyncNode->commitIndex);
@ -1743,7 +1758,10 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) {
} }
} }
ASSERT(b1 == b2); if (b1 != b2) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return false;
}
return b1; return b1;
} }
@ -2194,7 +2212,7 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {
} }
SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
// ASSERT(lastIndex >= 0);
sInfo("vgId:%d, become leader. term:%" PRId64 ", commit index:%" PRId64 ", last index:%" PRId64 "", pSyncNode->vgId, sInfo("vgId:%d, become leader. term:%" PRId64 ", commit index:%" PRId64 ", last index:%" PRId64 "", pSyncNode->vgId,
raftStoreGetTerm(pSyncNode), pSyncNode->commitIndex, lastIndex); raftStoreGetTerm(pSyncNode), pSyncNode->commitIndex, lastIndex);
} }
@ -2222,7 +2240,7 @@ void syncNodeFollower2Candidate(SSyncNode* pSyncNode) {
} }
int32_t syncNodeAssignedLeader2Leader(SSyncNode* pSyncNode) { int32_t syncNodeAssignedLeader2Leader(SSyncNode* pSyncNode) {
ASSERT(pSyncNode->state == TAOS_SYNC_STATE_ASSIGNED_LEADER); if (pSyncNode->state != TAOS_SYNC_STATE_ASSIGNED_LEADER) return TSDB_CODE_SYN_INTERNAL_ERROR;
syncNodeBecomeLeader(pSyncNode, "assigned leader to leader"); syncNodeBecomeLeader(pSyncNode, "assigned leader to leader");
sNTrace(pSyncNode, "assigned leader to leader"); sNTrace(pSyncNode, "assigned leader to leader");

127
source/libs/sync/src/syncPipeline.c
View File

@ -71,16 +71,32 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt
goto _err; goto _err;
} }
ASSERT(index == pBuf->endIndex); if (index != pBuf->endIndex) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _err;
};
SSyncRaftEntry* pExist = pBuf->entries[index % pBuf->size].pItem; SSyncRaftEntry* pExist = pBuf->entries[index % pBuf->size].pItem;
ASSERT(pExist == NULL); if (pExist != NULL) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _err;
}
// initial log buffer with at least one item, e.g. commitIndex // initial log buffer with at least one item, e.g. commitIndex
SSyncRaftEntry* pMatch = pBuf->entries[(index - 1 + pBuf->size) % pBuf->size].pItem; SSyncRaftEntry* pMatch = pBuf->entries[(index - 1 + pBuf->size) % pBuf->size].pItem;
ASSERTS(pMatch != NULL, "no matched log entry"); if (pMatch == NULL) {
ASSERT(pMatch->index + 1 == index); sError("vgId:%d, no matched log entry", pNode->vgId);
ASSERT(pMatch->term <= pEntry->term); code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _err;
}
if (pMatch->index + 1 != index) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _err;
}
if (!(pMatch->term <= pEntry->term)) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _err;
}
SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = pMatch->index, .prevLogTerm = pMatch->term}; SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = pMatch->index, .prevLogTerm = pMatch->term};
pBuf->entries[index % pBuf->size] = tmp; pBuf->entries[index % pBuf->size] = tmp;
@ -114,7 +130,7 @@ int32_t syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncI
TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST); TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
} }
ASSERT(index - 1 == prevIndex); if (index - 1 != prevIndex) return TSDB_CODE_SYN_INTERNAL_ERROR;
if (prevIndex >= pBuf->startIndex) { if (prevIndex >= pBuf->startIndex) {
pEntry = pBuf->entries[(prevIndex + pBuf->size) % pBuf->size].pItem; pEntry = pBuf->entries[(prevIndex + pBuf->size) % pBuf->size].pItem;
@ -178,9 +194,18 @@ int32_t syncLogValidateAlignmentOfCommit(SSyncNode* pNode, SyncIndex commitIndex
} }
int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) {
ASSERTS(pNode->pLogStore != NULL, "log store not created"); if (pNode->pLogStore == NULL) {
ASSERTS(pNode->pFsm != NULL, "pFsm not registered"); sError("log store not created");
ASSERTS(pNode->pFsm->FpGetSnapshotInfo != NULL, "FpGetSnapshotInfo not registered"); return TSDB_CODE_SYN_INTERNAL_ERROR;
}
if (pNode->pFsm == NULL) {
sError("pFsm not registered");
return TSDB_CODE_SYN_INTERNAL_ERROR;
}
if (pNode->pFsm->FpGetSnapshotInfo == NULL) {
sError("FpGetSnapshotInfo not registered");
return TSDB_CODE_SYN_INTERNAL_ERROR;
}
int32_t code = 0, lino = 0; int32_t code = 0, lino = 0;
SSnapshot snapshot = {0}; SSnapshot snapshot = {0};
@ -191,7 +216,8 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) {
TAOS_CHECK_EXIT(syncLogValidateAlignmentOfCommit(pNode, commitIndex)); TAOS_CHECK_EXIT(syncLogValidateAlignmentOfCommit(pNode, commitIndex));
SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore);
ASSERT(lastVer >= commitIndex); if (lastVer < commitIndex) return TSDB_CODE_SYN_INTERNAL_ERROR;
;
SyncIndex toIndex = lastVer; SyncIndex toIndex = lastVer;
// update match index // update match index
pBuf->commitIndex = commitIndex; pBuf->commitIndex = commitIndex;
@ -239,7 +265,7 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) {
// put a dummy record at commitIndex if present in log buffer // put a dummy record at commitIndex if present in log buffer
if (takeDummy) { if (takeDummy) {
ASSERT(index == pBuf->commitIndex); if (index != pBuf->commitIndex) return TSDB_CODE_SYN_INTERNAL_ERROR;
SSyncRaftEntry* pDummy = syncEntryBuildDummy(commitTerm, commitIndex, pNode->vgId); SSyncRaftEntry* pDummy = syncEntryBuildDummy(commitTerm, commitIndex, pNode->vgId);
if (pDummy == NULL) { if (pDummy == NULL) {
@ -392,9 +418,18 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt
} }
// update // update
ASSERT(pBuf->startIndex < index); if (!(pBuf->startIndex < index)) {
ASSERT(index - pBuf->startIndex < pBuf->size); code = TSDB_CODE_SYN_INTERNAL_ERROR;
ASSERT(pBuf->entries[index % pBuf->size].pItem == NULL); goto _out;
};
if (!(index - pBuf->startIndex < pBuf->size)) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _out;
}
if (pBuf->entries[index % pBuf->size].pItem != NULL) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _out;
}
SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = prevIndex, .prevLogTerm = prevTerm}; SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = prevIndex, .prevLogTerm = prevTerm};
pEntry = NULL; pEntry = NULL;
pBuf->entries[index % pBuf->size] = tmp; pBuf->entries[index % pBuf->size] = tmp;
@ -422,14 +457,14 @@ static inline bool syncLogStoreNeedFlush(SSyncRaftEntry* pEntry, int32_t replica
int32_t syncLogStorePersist(SSyncLogStore* pLogStore, SSyncNode* pNode, SSyncRaftEntry* pEntry) { int32_t syncLogStorePersist(SSyncLogStore* pLogStore, SSyncNode* pNode, SSyncRaftEntry* pEntry) {
int32_t code = 0; int32_t code = 0;
ASSERT(pEntry->index >= 0); if (pEntry->index < 0) return TSDB_CODE_SYN_INTERNAL_ERROR;
SyncIndex lastVer = pLogStore->syncLogLastIndex(pLogStore); SyncIndex lastVer = pLogStore->syncLogLastIndex(pLogStore);
if (lastVer >= pEntry->index && (code = pLogStore->syncLogTruncate(pLogStore, pEntry->index)) < 0) { if (lastVer >= pEntry->index && (code = pLogStore->syncLogTruncate(pLogStore, pEntry->index)) < 0) {
sError("failed to truncate log store since %s. from index:%" PRId64 "", tstrerror(code), pEntry->index); sError("failed to truncate log store since %s. from index:%" PRId64 "", tstrerror(code), pEntry->index);
TAOS_RETURN(code); TAOS_RETURN(code);
} }
lastVer = pLogStore->syncLogLastIndex(pLogStore); lastVer = pLogStore->syncLogLastIndex(pLogStore);
ASSERT(pEntry->index == lastVer + 1); if (pEntry->index != lastVer + 1) return TSDB_CODE_SYN_INTERNAL_ERROR;
bool doFsync = syncLogStoreNeedFlush(pEntry, pNode->replicaNum); bool doFsync = syncLogStoreNeedFlush(pEntry, pNode->replicaNum);
if ((code = pLogStore->syncLogAppendEntry(pLogStore, pEntry, doFsync)) < 0) { if ((code = pLogStore->syncLogAppendEntry(pLogStore, pEntry, doFsync)) < 0) {
@ -439,7 +474,7 @@ int32_t syncLogStorePersist(SSyncLogStore* pLogStore, SSyncNode* pNode, SSyncRaf
} }
lastVer = pLogStore->syncLogLastIndex(pLogStore); lastVer = pLogStore->syncLogLastIndex(pLogStore);
ASSERT(pEntry->index == lastVer); if (pEntry->index != lastVer) return TSDB_CODE_SYN_INTERNAL_ERROR;
return 0; return 0;
} }
@ -718,7 +753,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm
SyncIndex until = pBuf->commitIndex - TSDB_SYNC_LOG_BUFFER_RETENTION; SyncIndex until = pBuf->commitIndex - TSDB_SYNC_LOG_BUFFER_RETENTION;
for (SyncIndex index = pBuf->startIndex; index < until; index++) { for (SyncIndex index = pBuf->startIndex; index < until; index++) {
SSyncRaftEntry* pEntry = pBuf->entries[(index + pBuf->size) % pBuf->size].pItem; SSyncRaftEntry* pEntry = pBuf->entries[(index + pBuf->size) % pBuf->size].pItem;
ASSERT(pEntry != NULL); if (pEntry == NULL) return TSDB_CODE_SYN_INTERNAL_ERROR;
syncEntryDestroy(pEntry); syncEntryDestroy(pEntry);
(void)memset(&pBuf->entries[(index + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0])); (void)memset(&pBuf->entries[(index + pBuf->size) % pBuf->size], 0, sizeof(pBuf->entries[0]));
pBuf->startIndex = index + 1; pBuf->startIndex = index + 1;
@ -786,7 +821,10 @@ int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
for (SyncIndex index = pMgr->startIndex; index < pMgr->endIndex; index++) { for (SyncIndex index = pMgr->startIndex; index < pMgr->endIndex; index++) {
int64_t pos = index % pMgr->size; int64_t pos = index % pMgr->size;
ASSERT(!pMgr->states[pos].barrier || (index == pMgr->startIndex || index + 1 == pMgr->endIndex)); if (!(!pMgr->states[pos].barrier || (index == pMgr->startIndex || index + 1 == pMgr->endIndex))) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _out;
}
if (nowMs < pMgr->states[pos].timeMs + retryWaitMs) { if (nowMs < pMgr->states[pos].timeMs + retryWaitMs) {
break; break;
@ -809,7 +847,10 @@ int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
tstrerror(code), index, pDestId->addr); tstrerror(code), index, pDestId->addr);
goto _out; goto _out;
} }
ASSERT(barrier == pMgr->states[pos].barrier); if (barrier != pMgr->states[pos].barrier) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _out;
}
pMgr->states[pos].timeMs = nowMs; pMgr->states[pos].timeMs = nowMs;
pMgr->states[pos].term = term; pMgr->states[pos].term = term;
pMgr->states[pos].acked = false; pMgr->states[pos].acked = false;
@ -839,11 +880,11 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn
SSyncLogBuffer* pBuf = pNode->pLogBuf; SSyncLogBuffer* pBuf = pNode->pLogBuf;
SRaftId destId = pMsg->srcId; SRaftId destId = pMsg->srcId;
int32_t code = 0; int32_t code = 0;
ASSERT(pMgr->restored == false); if (pMgr->restored != false) return TSDB_CODE_SYN_INTERNAL_ERROR;
if (pMgr->endIndex == 0) { if (pMgr->endIndex == 0) {
ASSERT(pMgr->startIndex == 0); if (pMgr->startIndex != 0) return TSDB_CODE_SYN_INTERNAL_ERROR;
ASSERT(pMgr->matchIndex == 0); if (pMgr->matchIndex != 0) return TSDB_CODE_SYN_INTERNAL_ERROR;
if (pMsg->matchIndex < 0) { if (pMsg->matchIndex < 0) {
pMgr->restored = true; pMgr->restored = true;
sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64 sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 " %" PRId64 ", %" PRId64
@ -909,7 +950,7 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn
if ((index + 1 < firstVer) || (term < 0) || if ((index + 1 < firstVer) || (term < 0) ||
(term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) { (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) {
ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); if (!(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST)) return TSDB_CODE_SYN_INTERNAL_ERROR;
if ((code = syncNodeStartSnapshot(pNode, &destId)) < 0) { if ((code = syncNodeStartSnapshot(pNode, &destId)) < 0) {
sError("vgId:%d, failed to start snapshot for peer dnode:%d", pNode->vgId, DID(&destId)); sError("vgId:%d, failed to start snapshot for peer dnode:%d", pNode->vgId, DID(&destId));
TAOS_RETURN(code); TAOS_RETURN(code);
@ -918,13 +959,13 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn
return 0; return 0;
} }
ASSERT(index + 1 >= firstVer); if (!(index + 1 >= firstVer)) return TSDB_CODE_SYN_INTERNAL_ERROR;
if (term == pMsg->lastMatchTerm) { if (term == pMsg->lastMatchTerm) {
index = index + 1; index = index + 1;
ASSERT(index <= pNode->pLogBuf->matchIndex); if (!(index <= pNode->pLogBuf->matchIndex)) return TSDB_CODE_SYN_INTERNAL_ERROR;
} else { } else {
ASSERT(index > firstVer); if (!(index > firstVer)) return TSDB_CODE_SYN_INTERNAL_ERROR;
} }
} }
@ -975,8 +1016,8 @@ int32_t syncLogReplStart(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
} }
int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) {
ASSERT(!pMgr->restored); if (pMgr->restored) return TSDB_CODE_SYN_INTERNAL_ERROR;
ASSERT(pMgr->startIndex >= 0); if (!(pMgr->startIndex >= 0)) return TSDB_CODE_SYN_INTERNAL_ERROR;
int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs(); int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs();
int64_t nowMs = taosGetMonoTimestampMs(); int64_t nowMs = taosGetMonoTimestampMs();
int32_t code = 0; int32_t code = 0;
@ -996,7 +1037,7 @@ int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex inde
TAOS_RETURN(code); TAOS_RETURN(code);
} }
ASSERT(index >= 0); if (!(index >= 0)) return TSDB_CODE_SYN_INTERNAL_ERROR;
pMgr->states[index % pMgr->size].barrier = barrier; pMgr->states[index % pMgr->size].barrier = barrier;
pMgr->states[index % pMgr->size].timeMs = nowMs; pMgr->states[index % pMgr->size].timeMs = nowMs;
pMgr->states[index % pMgr->size].term = term; pMgr->states[index % pMgr->size].term = term;
@ -1014,7 +1055,7 @@ int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex inde
} }
int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
ASSERT(pMgr->restored); if (!pMgr->restored) return TSDB_CODE_SYN_INTERNAL_ERROR;
SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; SRaftId* pDestId = &pNode->replicasId[pMgr->peerId];
int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff)); int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff));
@ -1070,7 +1111,7 @@ int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
} }
int32_t syncLogReplContinue(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { int32_t syncLogReplContinue(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) {
ASSERT(pMgr->restored == true); if (pMgr->restored != true) return TSDB_CODE_SYN_INTERNAL_ERROR;
if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) {
if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) {
int64_t firstMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; int64_t firstMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs;
@ -1100,7 +1141,10 @@ SSyncLogReplMgr* syncLogReplCreate() {
pMgr->size = sizeof(pMgr->states) / sizeof(pMgr->states[0]); pMgr->size = sizeof(pMgr->states) / sizeof(pMgr->states[0]);
ASSERT(pMgr->size == TSDB_SYNC_LOG_BUFFER_SIZE); if (pMgr->size != TSDB_SYNC_LOG_BUFFER_SIZE) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return NULL;
}
return pMgr; return pMgr;
} }
@ -1115,7 +1159,7 @@ void syncLogReplDestroy(SSyncLogReplMgr* pMgr) {
int32_t syncNodeLogReplInit(SSyncNode* pNode) { int32_t syncNodeLogReplInit(SSyncNode* pNode) {
for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) { for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) {
ASSERT(pNode->logReplMgrs[i] == NULL); if (pNode->logReplMgrs[i] != NULL) return TSDB_CODE_SYN_INTERNAL_ERROR;
pNode->logReplMgrs[i] = syncLogReplCreate(); pNode->logReplMgrs[i] = syncLogReplCreate();
if (pNode->logReplMgrs[i] == NULL) { if (pNode->logReplMgrs[i] == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
@ -1141,7 +1185,10 @@ int32_t syncLogBufferCreate(SSyncLogBuffer** ppBuf) {
pBuf->size = sizeof(pBuf->entries) / sizeof(pBuf->entries[0]); pBuf->size = sizeof(pBuf->entries) / sizeof(pBuf->entries[0]);
ASSERT(pBuf->size == TSDB_SYNC_LOG_BUFFER_SIZE); if (pBuf->size != TSDB_SYNC_LOG_BUFFER_SIZE) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _exit;
}
if (taosThreadMutexAttrInit(&pBuf->attr) < 0) { if (taosThreadMutexAttrInit(&pBuf->attr) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
@ -1194,7 +1241,7 @@ void syncLogBufferDestroy(SSyncLogBuffer* pBuf) {
int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex) { int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex) {
int32_t code = 0; int32_t code = 0;
ASSERT(pBuf->commitIndex < toIndex && toIndex <= pBuf->endIndex); if (!(pBuf->commitIndex < toIndex && toIndex <= pBuf->endIndex)) return TSDB_CODE_SYN_INTERNAL_ERROR;
if (toIndex == pBuf->endIndex) { if (toIndex == pBuf->endIndex) {
return 0; return 0;
@ -1217,7 +1264,7 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex
} }
pBuf->endIndex = toIndex; pBuf->endIndex = toIndex;
pBuf->matchIndex = TMIN(pBuf->matchIndex, index); pBuf->matchIndex = TMIN(pBuf->matchIndex, index);
ASSERT(index + 1 == toIndex); if (index + 1 != toIndex) return TSDB_CODE_SYN_INTERNAL_ERROR;
// trunc wal // trunc wal
SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore);
@ -1227,7 +1274,7 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex
TAOS_RETURN(code); TAOS_RETURN(code);
} }
lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore);
ASSERT(toIndex == lastVer + 1); if (toIndex != lastVer + 1) return TSDB_CODE_SYN_INTERNAL_ERROR;
// refill buffer on need // refill buffer on need
if (toIndex <= pBuf->startIndex) { if (toIndex <= pBuf->startIndex) {
@ -1237,7 +1284,7 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex
} }
} }
ASSERT(pBuf->endIndex == toIndex); if (pBuf->endIndex != toIndex) return TSDB_CODE_SYN_INTERNAL_ERROR;
(void)syncLogBufferValidate(pBuf); (void)syncLogBufferValidate(pBuf);
return 0; return 0;
} }
@ -1246,7 +1293,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) {
(void)taosThreadMutexLock(&pBuf->mutex); (void)taosThreadMutexLock(&pBuf->mutex);
(void)syncLogBufferValidate(pBuf); (void)syncLogBufferValidate(pBuf);
SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore);
ASSERT(lastVer == pBuf->matchIndex); if (lastVer != pBuf->matchIndex) return TSDB_CODE_SYN_INTERNAL_ERROR;
SyncIndex index = pBuf->endIndex - 1; SyncIndex index = pBuf->endIndex - 1;
(void)syncLogBufferRollback(pBuf, pNode, pBuf->matchIndex + 1); (void)syncLogBufferRollback(pBuf, pNode, pBuf->matchIndex + 1);

5
source/libs/sync/src/syncRaftEntry.c
View File

@ -70,7 +70,10 @@ SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) {
return NULL; return NULL;
} }
memcpy(pEntry, pMsg->data, pMsg->dataLen); memcpy(pEntry, pMsg->data, pMsg->dataLen);
ASSERT(pEntry->bytes == pMsg->dataLen); if (pEntry->bytes != pMsg->dataLen) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return NULL;
}
return pEntry; return pEntry;
} }

15
source/libs/sync/src/syncRaftLog.c
View File

@ -61,7 +61,10 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
pData->pSyncNode = pSyncNode; pData->pSyncNode = pSyncNode;
pData->pWal = pSyncNode->pWal; pData->pWal = pSyncNode->pWal;
ASSERT(pData->pWal != NULL); if (pData->pWal == NULL) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return NULL;
}
(void)taosThreadMutexInit(&(pData->mutex), NULL); (void)taosThreadMutexInit(&(pData->mutex), NULL);
pData->pWalHandle = walOpenReader(pData->pWal, NULL, 0); pData->pWalHandle = walOpenReader(pData->pWal, NULL, 0);
@ -115,7 +118,7 @@ void logStoreDestory(SSyncLogStore* pLogStore) {
// log[m .. n] // log[m .. n]
static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex) { static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex) {
ASSERT(snapshotIndex >= 0); if (!(snapshotIndex >= 0)) return TSDB_CODE_SYN_INTERNAL_ERROR;
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
@ -303,14 +306,14 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR
} }
*ppEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); *ppEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
ASSERT(*ppEntry != NULL); if (*ppEntry == NULL) return TSDB_CODE_SYN_INTERNAL_ERROR;
(*ppEntry)->msgType = TDMT_SYNC_CLIENT_REQUEST; (*ppEntry)->msgType = TDMT_SYNC_CLIENT_REQUEST;
(*ppEntry)->originalRpcType = pWalHandle->pHead->head.msgType; (*ppEntry)->originalRpcType = pWalHandle->pHead->head.msgType;
(*ppEntry)->seqNum = pWalHandle->pHead->head.syncMeta.seqNum; (*ppEntry)->seqNum = pWalHandle->pHead->head.syncMeta.seqNum;
(*ppEntry)->isWeak = pWalHandle->pHead->head.syncMeta.isWeek; (*ppEntry)->isWeak = pWalHandle->pHead->head.syncMeta.isWeek;
(*ppEntry)->term = pWalHandle->pHead->head.syncMeta.term; (*ppEntry)->term = pWalHandle->pHead->head.syncMeta.term;
(*ppEntry)->index = index; (*ppEntry)->index = index;
ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen); if ((*ppEntry)->dataLen != pWalHandle->pHead->head.bodyLen) return TSDB_CODE_SYN_INTERNAL_ERROR;
(void)memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); (void)memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen);
/* /*
@ -362,14 +365,14 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn
static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
ASSERT(ppLastEntry != NULL); if (ppLastEntry == NULL) return TSDB_CODE_SYN_INTERNAL_ERROR;
*ppLastEntry = NULL; *ppLastEntry = NULL;
if (walIsEmpty(pWal)) { if (walIsEmpty(pWal)) {
TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST); TAOS_RETURN(TSDB_CODE_WAL_LOG_NOT_EXIST);
} else { } else {
SyncIndex lastIndex = raftLogLastIndex(pLogStore); SyncIndex lastIndex = raftLogLastIndex(pLogStore);
ASSERT(lastIndex >= SYNC_INDEX_BEGIN); if (!(lastIndex >= SYNC_INDEX_BEGIN)) return TSDB_CODE_SYN_INTERNAL_ERROR;
int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry);
TAOS_RETURN(code); TAOS_RETURN(code);

4
source/libs/sync/src/syncRequestVote.c
View File

@ -104,7 +104,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
syncNodeStepDown(ths, pMsg->term); syncNodeStepDown(ths, pMsg->term);
} }
SyncTerm currentTerm = raftStoreGetTerm(ths); SyncTerm currentTerm = raftStoreGetTerm(ths);
ASSERT(pMsg->term <= currentTerm); if (!(pMsg->term <= currentTerm)) return TSDB_CODE_SYN_INTERNAL_ERROR;
bool grant = (pMsg->term == currentTerm) && logOK && bool grant = (pMsg->term == currentTerm) && logOK &&
((!raftStoreHasVoted(ths)) || (syncUtilSameId(&ths->raftStore.voteFor, &pMsg->srcId))); ((!raftStoreHasVoted(ths)) || (syncUtilSameId(&ths->raftStore.voteFor, &pMsg->srcId)));
@ -130,7 +130,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
pReply->destId = pMsg->srcId; pReply->destId = pMsg->srcId;
pReply->term = currentTerm; pReply->term = currentTerm;
pReply->voteGranted = grant; pReply->voteGranted = grant;
ASSERT(!grant || pMsg->term == pReply->term); if (!(!grant || pMsg->term == pReply->term)) return TSDB_CODE_SYN_INTERNAL_ERROR;
// trace log // trace log
syncLogRecvRequestVote(ths, pMsg, pReply->voteGranted, ""); syncLogRecvRequestVote(ths, pMsg, pReply->voteGranted, "");

2
source/libs/sync/src/syncRequestVoteReply.c
View File

@ -64,7 +64,7 @@ int32_t syncNodeOnRequestVoteReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
} }
syncLogRecvRequestVoteReply(ths, pMsg, ""); syncLogRecvRequestVoteReply(ths, pMsg, "");
ASSERT(pMsg->term == currentTerm); if (pMsg->term != currentTerm) return TSDB_CODE_SYN_INTERNAL_ERROR;
// This tallies votes even when the current state is not Candidate, // This tallies votes even when the current state is not Candidate,
// but they won't be looked at, so it doesn't matter. // but they won't be looked at, so it doesn't matter.

35
source/libs/sync/src/syncSnapshot.c
View File

@ -57,7 +57,7 @@ static int32_t syncSnapBufferCreate(SSyncSnapBuffer **ppBuf) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
} }
pBuf->size = sizeof(pBuf->entries) / sizeof(void *); pBuf->size = sizeof(pBuf->entries) / sizeof(void *);
ASSERT(pBuf->size == TSDB_SYNC_SNAP_BUFFER_SIZE); if (pBuf->size != TSDB_SYNC_SNAP_BUFFER_SIZE) return TSDB_CODE_SYN_INTERNAL_ERROR;
(void)taosThreadMutexInit(&pBuf->mutex, NULL); (void)taosThreadMutexInit(&pBuf->mutex, NULL);
*ppBuf = pBuf; *ppBuf = pBuf;
TAOS_RETURN(0); TAOS_RETURN(0);
@ -311,7 +311,10 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) {
} }
} }
ASSERT(pSender->seq >= SYNC_SNAPSHOT_SEQ_BEGIN && pSender->seq <= SYNC_SNAPSHOT_SEQ_END); if (!(pSender->seq >= SYNC_SNAPSHOT_SEQ_BEGIN && pSender->seq <= SYNC_SNAPSHOT_SEQ_END)) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _OUT;
}
// send msg // send msg
int32_t blockLen = (pBlk) ? pBlk->blockLen : 0; int32_t blockLen = (pBlk) ? pBlk->blockLen : 0;
@ -323,7 +326,10 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) {
// put in buffer // put in buffer
int64_t nowMs = taosGetTimestampMs(); int64_t nowMs = taosGetTimestampMs();
if (pBlk) { if (pBlk) {
ASSERT(pBlk->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pBlk->seq < SYNC_SNAPSHOT_SEQ_END); if (!(pBlk->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pBlk->seq < SYNC_SNAPSHOT_SEQ_END)) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _OUT;
}
pBlk->sendTimeMs = nowMs; pBlk->sendTimeMs = nowMs;
pSender->pSndBuf->entries[pSender->seq % pSender->pSndBuf->size] = pBlk; pSender->pSndBuf->entries[pSender->seq % pSender->pSndBuf->size] = pBlk;
pBlk = NULL; pBlk = NULL;
@ -351,7 +357,10 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
for (int32_t seq = pSndBuf->cursor + 1; seq < pSndBuf->end; ++seq) { for (int32_t seq = pSndBuf->cursor + 1; seq < pSndBuf->end; ++seq) {
SyncSnapBlock *pBlk = pSndBuf->entries[seq % pSndBuf->size]; SyncSnapBlock *pBlk = pSndBuf->entries[seq % pSndBuf->size];
ASSERT(pBlk); if (!pBlk) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _out;
}
int64_t nowMs = taosGetTimestampMs(); int64_t nowMs = taosGetTimestampMs();
if (pBlk->acked || nowMs < pBlk->sendTimeMs + SYNC_SNAP_RESEND_MS) { if (pBlk->acked || nowMs < pBlk->sendTimeMs + SYNC_SNAP_RESEND_MS) {
continue; continue;
@ -682,7 +691,7 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) {
static int32_t syncSnapReceiverExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver, static int32_t syncSnapReceiverExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver,
SyncSnapshotSend *pMsg, SSnapshot *pInfo) { SyncSnapshotSend *pMsg, SSnapshot *pInfo) {
ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT); if (pMsg->payloadType != TDMT_SYNC_PREP_SNAPSHOT) return TSDB_CODE_SYN_INTERNAL_ERROR;
int32_t code = 0, lino = 0; int32_t code = 0, lino = 0;
// copy snap info from leader // copy snap info from leader
@ -878,7 +887,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot
goto _out; goto _out;
} }
ASSERT(pRcvBuf->start <= pRcvBuf->cursor + 1 && pRcvBuf->cursor < pRcvBuf->end); if (!(pRcvBuf->start <= pRcvBuf->cursor + 1 && pRcvBuf->cursor < pRcvBuf->end)) return TSDB_CODE_SYN_INTERNAL_ERROR;
if (pMsg->seq > pRcvBuf->cursor) { if (pMsg->seq > pRcvBuf->cursor) {
if (pRcvBuf->entries[pMsg->seq % pRcvBuf->size]) { if (pRcvBuf->entries[pMsg->seq % pRcvBuf->size]) {
@ -922,7 +931,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend
// condition 4 // condition 4
// transfering // transfering
SyncSnapshotSend *pMsg = ppMsg[0]; SyncSnapshotSend *pMsg = ppMsg[0];
ASSERT(pMsg); if (!pMsg) return TSDB_CODE_SYN_INTERNAL_ERROR;
SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
int64_t timeNow = taosGetTimestampMs(); int64_t timeNow = taosGetTimestampMs();
int32_t code = 0; int32_t code = 0;
@ -1071,7 +1080,7 @@ _out:;
} }
static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY); if (pMsg->payloadType != TDMT_SYNC_PREP_SNAPSHOT_REPLY) return TSDB_CODE_SYN_INTERNAL_ERROR;
SSyncTLV *datHead = (void *)pMsg->data; SSyncTLV *datHead = (void *)pMsg->data;
if (datHead->typ != pMsg->payloadType) { if (datHead->typ != pMsg->payloadType) {
@ -1168,11 +1177,17 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp
goto _out; goto _out;
} }
ASSERT(pSndBuf->start <= pSndBuf->cursor + 1 && pSndBuf->cursor < pSndBuf->end); if (!(pSndBuf->start <= pSndBuf->cursor + 1 && pSndBuf->cursor < pSndBuf->end)) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _out;
}
if (pMsg->ack > pSndBuf->cursor && pMsg->ack < pSndBuf->end) { if (pMsg->ack > pSndBuf->cursor && pMsg->ack < pSndBuf->end) {
SyncSnapBlock *pBlk = pSndBuf->entries[pMsg->ack % pSndBuf->size]; SyncSnapBlock *pBlk = pSndBuf->entries[pMsg->ack % pSndBuf->size];
ASSERT(pBlk); if (!pBlk) {
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _out;
}
pBlk->acked = 1; pBlk->acked = 1;
} }

2
source/libs/tdb/src/db/tdbBtree.c
View File

@ -1133,7 +1133,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const
memcpy(pCell + nLocal - sizeof(pgno), &pgno, sizeof(pgno)); memcpy(pCell + nLocal - sizeof(pgno), &pgno, sizeof(pgno));
int lastKeyPageSpace = 0; size_t lastKeyPageSpace = 0;
// pack left key & val to ovpages // pack left key & val to ovpages
do { do {
// cal key to cpy // cal key to cpy

12
source/libs/transport/inc/transLog.h
View File

@ -32,12 +32,12 @@ extern "C" {
#define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }} #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }}
#define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } } #define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } }
#define tGTrace(param, ...) do { if (rpcDebugFlag & DEBUG_TRACE){char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define tGTrace(param, ...) do { if (rpcDebugFlag & DEBUG_TRACE){char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define tGFatal(param, ...) do {if (rpcDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tFatal(param ", gtid:%s", __VA_ARGS__, buf); }} while (0) #define tGFatal(param, ...) do {if (rpcDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tFatal(param ", QID:%s", __VA_ARGS__, buf); }} while (0)
#define tGError(param, ...) do { if (rpcDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tError(param ", gtid:%s", __VA_ARGS__, buf);} } while(0) #define tGError(param, ...) do { if (rpcDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tError(param ", QID:%s", __VA_ARGS__, buf);} } while(0)
#define tGWarn(param, ...) do { if (rpcDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tWarn(param ", gtid:%s", __VA_ARGS__, buf); }} while(0) #define tGWarn(param, ...) do { if (rpcDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tWarn(param ", QID:%s", __VA_ARGS__, buf); }} while(0)
#define tGInfo(param, ...) do { if (rpcDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tInfo(param ", gtid:%s", __VA_ARGS__, buf); }} while(0) #define tGInfo(param, ...) do { if (rpcDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tInfo(param ", QID:%s", __VA_ARGS__, buf); }} while(0)
#define tGDebug(param,...) do {if (rpcDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tDebug(param ", gtid:%s", __VA_ARGS__, buf); }} while(0) #define tGDebug(param,...) do {if (rpcDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tDebug(param ", QID:%s", __VA_ARGS__, buf); }} while(0)
// clang-format on // clang-format on

9
source/os/src/osFile.c
View File

@ -1633,7 +1633,14 @@ int taosCloseCFile(FILE *f) { return fclose(f); }
int taosSetAutoDelFile(char *path) { int taosSetAutoDelFile(char *path) {
#ifdef WINDOWS #ifdef WINDOWS
return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY); bool succ = SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY);
if (succ) {
return 0;
} else {
DWORD error = GetLastError();
terrno = TAOS_SYSTEM_ERROR(error);
return terrno;
}
#else #else
if (-1 == unlink(path)) { if (-1 == unlink(path)) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);

29
source/util/src/tarray.c
View File

@ -266,8 +266,9 @@ void* taosArrayInsert(SArray* pArray, size_t index, const void* pData) {
} }
void taosArraySet(SArray* pArray, size_t index, void* pData) { void taosArraySet(SArray* pArray, size_t index, void* pData) {
ASSERT(index < pArray->size); if (index < pArray->size) {
memcpy(TARRAY_GET_ELEM(pArray, index), pData, pArray->elemSize); memcpy(TARRAY_GET_ELEM(pArray, index), pData, pArray->elemSize);
}
} }
void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) {
@ -291,7 +292,9 @@ void taosArrayPopTailBatch(SArray* pArray, size_t cnt) {
} }
void taosArrayRemove(SArray* pArray, size_t index) { void taosArrayRemove(SArray* pArray, size_t index) {
ASSERT(index < pArray->size); if (!(index < pArray->size)) {
return;
}
if (index == pArray->size - 1) { if (index == pArray->size - 1) {
(void)taosArrayPop(pArray); (void)taosArrayPop(pArray);
@ -305,17 +308,17 @@ void taosArrayRemove(SArray* pArray, size_t index) {
} }
void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp) { void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp) {
ASSERT(index + num <= pArray->size); if (index + num <= pArray->size) {
if (fp) {
if (fp) { for (int32_t i = 0; i < num; i++) {
for (int32_t i = 0; i < num; i++) { fp(taosArrayGet(pArray, index + i));
fp(taosArrayGet(pArray, index + i)); }
} }
}
memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + num) * pArray->elemSize, memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + num) * pArray->elemSize,
(pArray->size - index - num) * pArray->elemSize); (pArray->size - index - num) * pArray->elemSize);
pArray->size -= num; pArray->size -= num;
}
} }
SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) {
@ -349,8 +352,6 @@ SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) {
if (fn == NULL) { if (fn == NULL) {
memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size);
} else { } else {
ASSERT(pSrc->elemSize == sizeof(void*));
for (int32_t i = 0; i < pSrc->size; ++i) { for (int32_t i = 0; i < pSrc->size; ++i) {
void* p = fn(taosArrayGetP(pSrc, i)); void* p = fn(taosArrayGetP(pSrc, i));
memcpy(((char*)dst->pData) + i * dst->elemSize, &p, dst->elemSize); memcpy(((char*)dst->pData) + i * dst->elemSize, &p, dst->elemSize);

4
source/util/src/tbloomfilter.c
View File

@ -78,7 +78,9 @@ _error:
} }
int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) { int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) {
ASSERT(!tBloomFilterIsFull(pBF)); if (tBloomFilterIsFull(pBF)) {
return TSDB_CODE_FAILED;
}
bool hasChange = false; bool hasChange = false;
const register uint64_t size = pBF->numBits; const register uint64_t size = pBF->numBits;
uint64_t cbHash = hash1; uint64_t cbHash = hash1;

1168
source/util/src/tcompression.c
View File

File diff suppressed because it is too large Load Diff

4
source/util/src/tconfig.c
View File

@ -121,7 +121,9 @@ int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->array); }
static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) { static int32_t cfgCheckAndSetConf(SConfigItem *pItem, const char *conf) {
cfgItemFreeVal(pItem); cfgItemFreeVal(pItem);
ASSERT(pItem->str == NULL); if (!(pItem->str == NULL)) {
return TSDB_CODE_INVALID_PARA;
}
pItem->str = taosStrdup(conf); pItem->str = taosStrdup(conf);
if (pItem->str == NULL) { if (pItem->str == NULL) {

4
source/util/src/tdigest.c
View File

@ -143,24 +143,20 @@ int32_t tdigestCompress(TDigest *t) {
if (a->mean <= b->mean) { if (a->mean <= b->mean) {
mergeCentroid(&args, a); mergeCentroid(&args, a);
ASSERTS(args.idx < t->size, "idx over size");
i++; i++;
} else { } else {
mergeCentroid(&args, b); mergeCentroid(&args, b);
ASSERTS(args.idx < t->size, "idx over size");
j++; j++;
} }
} }
while (i < num_unmerged) { while (i < num_unmerged) {
mergeCentroid(&args, &unmerged_centroids[i++]); mergeCentroid(&args, &unmerged_centroids[i++]);
ASSERTS(args.idx < t->size, "idx over size");
} }
taosMemoryFree((void *)unmerged_centroids); taosMemoryFree((void *)unmerged_centroids);
while (j < t->num_centroids) { while (j < t->num_centroids) {
mergeCentroid(&args, &t->centroids[j++]); mergeCentroid(&args, &t->centroids[j++]);
ASSERTS(args.idx < t->size, "idx over size");
} }
if (t->total_weight > 0) { if (t->total_weight > 0) {

2
source/util/src/tencode.c
View File

@ -104,7 +104,6 @@ void tEndEncode(SEncoder* pCoder) {
if (pCoder->data) { if (pCoder->data) {
pNode = pCoder->eStack; pNode = pCoder->eStack;
ASSERT(pNode);
pCoder->eStack = pNode->pNext; pCoder->eStack = pNode->pNext;
len = pCoder->pos; len = pCoder->pos;
@ -148,7 +147,6 @@ void tEndDecode(SDecoder* pCoder) {
SDecoderNode* pNode; SDecoderNode* pNode;
pNode = pCoder->dStack; pNode = pCoder->dStack;
ASSERT(pNode);
pCoder->dStack = pNode->pNext; pCoder->dStack = pNode->pNext;
pCoder->data = pNode->data; pCoder->data = pNode->data;

150
source/util/src/texception.c
View File

@ -1,150 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "texception.h"
#include "tlog.h"
static threadlocal SExceptionNode* expList;
void exceptionPushNode(SExceptionNode* node) {
node->prev = expList;
expList = node;
}
int32_t exceptionPopNode() {
SExceptionNode* node = expList;
expList = node->prev;
return node->code;
}
void exceptionThrow(int32_t code) {
expList->code = code;
longjmp(expList->jb, 1);
}
static void cleanupWrapper_void_ptr_ptr(SCleanupAction* ca) {
void (*func)(void*, void*) = ca->func;
func(ca->arg1.Ptr, ca->arg2.Ptr);
}
static void cleanupWrapper_void_ptr_bool(SCleanupAction* ca) {
void (*func)(void*, bool) = ca->func;
func(ca->arg1.Ptr, ca->arg2.Bool);
}
static void cleanupWrapper_void_ptr(SCleanupAction* ca) {
void (*func)(void*) = ca->func;
func(ca->arg1.Ptr);
}
static void cleanupWrapper_int_int(SCleanupAction* ca) {
int32_t (*func)(int32_t) = ca->func;
(void)func(ca->arg1.Int);
}
static void cleanupWrapper_void(SCleanupAction* ca) {
void (*func)() = ca->func;
func();
}
static void cleanupWrapper_int_ptr(SCleanupAction* ca) {
int32_t (*func)(void*) = ca->func;
(void)func(ca->arg1.Ptr);
}
typedef void (*wrapper)(SCleanupAction*);
static wrapper wrappers[] = {
cleanupWrapper_void_ptr_ptr, cleanupWrapper_void_ptr_bool, cleanupWrapper_void_ptr,
cleanupWrapper_int_int, cleanupWrapper_void, cleanupWrapper_int_ptr,
};
void cleanupPush_void_ptr_ptr(bool failOnly, void* func, void* arg1, void* arg2) {
ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction");
SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 0;
ca->failOnly = failOnly;
ca->func = func;
ca->arg1.Ptr = arg1;
ca->arg2.Ptr = arg2;
}
void cleanupPush_void_ptr_bool(bool failOnly, void* func, void* arg1, bool arg2) {
ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction");
SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 1;
ca->failOnly = failOnly;
ca->func = func;
ca->arg1.Ptr = arg1;
ca->arg2.Bool = arg2;
}
void cleanupPush_void_ptr(bool failOnly, void* func, void* arg) {
ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction");
SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 2;
ca->failOnly = failOnly;
ca->func = func;
ca->arg1.Ptr = arg;
}
void cleanupPush_int_int(bool failOnly, void* func, int32_t arg) {
ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction");
SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 3;
ca->failOnly = failOnly;
ca->func = func;
ca->arg1.Int = arg;
}
void cleanupPush_void(bool failOnly, void* func) {
ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction");
SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 4;
ca->failOnly = failOnly;
ca->func = func;
}
void cleanupPush_int_ptr(bool failOnly, void* func, void* arg) {
ASSERTS(expList->numCleanupAction < expList->maxCleanupAction, "numCleanupAction over maxCleanupAction");
SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++;
ca->wrapper = 5;
ca->failOnly = failOnly;
ca->func = func;
ca->arg1.Ptr = arg;
}
int32_t cleanupGetActionCount() { return expList->numCleanupAction; }
static void doExecuteCleanup(SExceptionNode* node, int32_t anchor, bool failed) {
while (node->numCleanupAction > anchor) {
--node->numCleanupAction;
SCleanupAction* ca = node->cleanupActions + node->numCleanupAction;
if (failed || !(ca->failOnly)) {
wrappers[ca->wrapper](ca);
}
}
}
void cleanupExecuteTo(int32_t anchor, bool failed) { doExecuteCleanup(expList, anchor, failed); }
void cleanupExecute(SExceptionNode* node, bool failed) { doExecuteCleanup(node, 0, failed); }
bool cleanupExceedLimit() { return expList->numCleanupAction >= expList->maxCleanupAction; }

5
source/util/src/thash.c
View File

@ -191,14 +191,12 @@ static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SH
(void)atomic_sub_fetch_16(&pNode->refCount, 1); (void)atomic_sub_fetch_16(&pNode->refCount, 1);
if (prev != NULL) { if (prev != NULL) {
prev->next = pNewNode; prev->next = pNewNode;
ASSERT(prev->next != prev);
} else { } else {
pe->next = pNewNode; pe->next = pNewNode;
} }
if (pNode->refCount <= 0) { if (pNode->refCount <= 0) {
pNewNode->next = pNode->next; pNewNode->next = pNode->next;
ASSERT(pNewNode->next != pNewNode);
FREE_HASH_NODE(pHashObj->freeFp, pNode); FREE_HASH_NODE(pHashObj->freeFp, pNode);
} else { } else {
@ -508,7 +506,6 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
pe->next = pNode->next; pe->next = pNode->next;
} else { } else {
prevNode->next = pNode->next; prevNode->next = pNode->next;
ASSERT(prevNode->next != prevNode);
} }
pe->num--; pe->num--;
@ -759,12 +756,10 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
if (pOld->refCount <= 0) { if (pOld->refCount <= 0) {
if (prevNode) { if (prevNode) {
prevNode->next = pOld->next; prevNode->next = pOld->next;
ASSERT(prevNode->next != prevNode);
} else { } else {
pe->next = pOld->next; pe->next = pOld->next;
SHashNode *x = pe->next; SHashNode *x = pe->next;
if (x != NULL) { if (x != NULL) {
ASSERT(x->next != x);
} }
} }

57
source/util/src/tlrucache.c
View File

@ -87,13 +87,13 @@ struct SLRUEntry {
#define TAOS_LRU_ENTRY_REF(h) (++(h)->refs) #define TAOS_LRU_ENTRY_REF(h) (++(h)->refs)
static bool taosLRUEntryUnref(SLRUEntry *entry) { static bool taosLRUEntryUnref(SLRUEntry *entry) {
ASSERT(entry->refs > 0); // ASSERT(entry->refs > 0);
--entry->refs; --entry->refs;
return entry->refs == 0; return entry->refs == 0;
} }
static void taosLRUEntryFree(SLRUEntry *entry) { static void taosLRUEntryFree(SLRUEntry *entry) {
ASSERT(entry->refs == 0); // ASSERT(entry->refs == 0);
if (entry->deleter) { if (entry->deleter) {
(*entry->deleter)(entry->keyData, entry->keyLength, entry->value, entry->ud); (*entry->deleter)(entry->keyData, entry->keyLength, entry->value, entry->ud);
@ -129,7 +129,7 @@ static void taosLRUEntryTableApply(SLRUEntryTable *table, _taos_lru_table_func_t
SLRUEntry *h = table->list[i]; SLRUEntry *h = table->list[i];
while (h) { while (h) {
SLRUEntry *n = h->nextHash; SLRUEntry *n = h->nextHash;
ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h)); // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h));
func(h); func(h);
h = n; h = n;
} }
@ -155,7 +155,7 @@ static int taosLRUEntryTableApplyF(SLRUEntryTable *table, _taos_lru_functor_t fu
SLRUEntry *h = table->list[i]; SLRUEntry *h = table->list[i];
while (h) { while (h) {
SLRUEntry *n = h->nextHash; SLRUEntry *n = h->nextHash;
ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h)); // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h));
ret = functor(h->keyData, h->keyLength, h->value, ud); ret = functor(h->keyData, h->keyLength, h->value, ud);
if (ret) { if (ret) {
return ret; return ret;
@ -205,7 +205,7 @@ static void taosLRUEntryTableResize(SLRUEntryTable *table) {
++count; ++count;
} }
} }
ASSERT(table->elems == count); // ASSERT(table->elems == count);
taosMemoryFree(table->list); taosMemoryFree(table->list);
table->list = newList; table->list = newList;
@ -261,16 +261,16 @@ struct SLRUCacheShard {
static void taosLRUCacheShardMaintainPoolSize(SLRUCacheShard *shard) { static void taosLRUCacheShardMaintainPoolSize(SLRUCacheShard *shard) {
while (shard->highPriPoolUsage > shard->highPriPoolCapacity) { while (shard->highPriPoolUsage > shard->highPriPoolCapacity) {
shard->lruLowPri = shard->lruLowPri->next; shard->lruLowPri = shard->lruLowPri->next;
ASSERT(shard->lruLowPri != &shard->lru); // ASSERT(shard->lruLowPri != &shard->lru);
TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(shard->lruLowPri, false); TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(shard->lruLowPri, false);
ASSERT(shard->highPriPoolUsage >= shard->lruLowPri->totalCharge); // ASSERT(shard->highPriPoolUsage >= shard->lruLowPri->totalCharge);
shard->highPriPoolUsage -= shard->lruLowPri->totalCharge; shard->highPriPoolUsage -= shard->lruLowPri->totalCharge;
} }
} }
static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) { static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) {
ASSERT(e->next == NULL && e->prev == NULL); // ASSERT(e->next == NULL && e->prev == NULL);
if (shard->highPriPoolRatio > 0 && (TAOS_LRU_ENTRY_IS_HIGH_PRI(e) || TAOS_LRU_ENTRY_HAS_HIT(e))) { if (shard->highPriPoolRatio > 0 && (TAOS_LRU_ENTRY_IS_HIGH_PRI(e) || TAOS_LRU_ENTRY_HAS_HIT(e))) {
e->next = &shard->lru; e->next = &shard->lru;
@ -297,7 +297,7 @@ static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) {
} }
static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) { static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) {
ASSERT(e->next && e->prev); // ASSERT(e->next && e->prev);
if (shard->lruLowPri == e) { if (shard->lruLowPri == e) {
shard->lruLowPri = e->prev; shard->lruLowPri = e->prev;
@ -306,10 +306,10 @@ static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) {
e->prev->next = e->next; e->prev->next = e->next;
e->prev = e->next = NULL; e->prev = e->next = NULL;
ASSERT(shard->lruUsage >= e->totalCharge); // ASSERT(shard->lruUsage >= e->totalCharge);
shard->lruUsage -= e->totalCharge; shard->lruUsage -= e->totalCharge;
if (TAOS_LRU_ENTRY_IN_HIGH_POOL(e)) { if (TAOS_LRU_ENTRY_IN_HIGH_POOL(e)) {
ASSERT(shard->highPriPoolUsage >= e->totalCharge); // ASSERT(shard->highPriPoolUsage >= e->totalCharge);
shard->highPriPoolUsage -= e->totalCharge; shard->highPriPoolUsage -= e->totalCharge;
} }
} }
@ -317,13 +317,13 @@ static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) {
static void taosLRUCacheShardEvictLRU(SLRUCacheShard *shard, size_t charge, SArray *deleted) { static void taosLRUCacheShardEvictLRU(SLRUCacheShard *shard, size_t charge, SArray *deleted) {
while (shard->usage + charge > shard->capacity && shard->lru.next != &shard->lru) { while (shard->usage + charge > shard->capacity && shard->lru.next != &shard->lru) {
SLRUEntry *old = shard->lru.next; SLRUEntry *old = shard->lru.next;
ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old));
taosLRUCacheShardLRURemove(shard, old); taosLRUCacheShardLRURemove(shard, old);
(void)taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash); (void)taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash);
TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false);
ASSERT(shard->usage >= old->totalCharge); // ASSERT(shard->usage >= old->totalCharge);
shard->usage -= old->totalCharge; shard->usage -= old->totalCharge;
(void)taosArrayPush(deleted, &old); (void)taosArrayPush(deleted, &old);
@ -332,6 +332,9 @@ static void taosLRUCacheShardEvictLRU(SLRUCacheShard *shard, size_t charge, SArr
static void taosLRUCacheShardSetCapacity(SLRUCacheShard *shard, size_t capacity) { static void taosLRUCacheShardSetCapacity(SLRUCacheShard *shard, size_t capacity) {
SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES);
if (!lastReferenceList) {
return;
}
(void)taosThreadMutexLock(&shard->mutex); (void)taosThreadMutexLock(&shard->mutex);
@ -411,11 +414,11 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *
if (old != NULL) { if (old != NULL) {
status = TAOS_LRU_STATUS_OK_OVERWRITTEN; status = TAOS_LRU_STATUS_OK_OVERWRITTEN;
ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old)); // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old));
TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false);
if (!TAOS_LRU_ENTRY_HAS_REFS(old)) { if (!TAOS_LRU_ENTRY_HAS_REFS(old)) {
taosLRUCacheShardLRURemove(shard, old); taosLRUCacheShardLRURemove(shard, old);
ASSERT(shard->usage >= old->totalCharge); // ASSERT(shard->usage >= old->totalCharge);
shard->usage -= old->totalCharge; shard->usage -= old->totalCharge;
(void)taosArrayPush(lastReferenceList, &old); (void)taosArrayPush(lastReferenceList, &old);
@ -476,7 +479,7 @@ static LRUHandle *taosLRUCacheShardLookup(SLRUCacheShard *shard, const void *key
(void)taosThreadMutexLock(&shard->mutex); (void)taosThreadMutexLock(&shard->mutex);
e = taosLRUEntryTableLookup(&shard->table, key, keyLen, hash); e = taosLRUEntryTableLookup(&shard->table, key, keyLen, hash);
if (e != NULL) { if (e != NULL) {
ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e)); // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e));
if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { if (!TAOS_LRU_ENTRY_HAS_REFS(e)) {
taosLRUCacheShardLRURemove(shard, e); taosLRUCacheShardLRURemove(shard, e);
} }
@ -495,12 +498,12 @@ static void taosLRUCacheShardErase(SLRUCacheShard *shard, const void *key, size_
SLRUEntry *e = taosLRUEntryTableRemove(&shard->table, key, keyLen, hash); SLRUEntry *e = taosLRUEntryTableRemove(&shard->table, key, keyLen, hash);
if (e != NULL) { if (e != NULL) {
ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e)); // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e));
TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); TAOS_LRU_ENTRY_SET_IN_CACHE(e, false);
if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { if (!TAOS_LRU_ENTRY_HAS_REFS(e)) {
taosLRUCacheShardLRURemove(shard, e); taosLRUCacheShardLRURemove(shard, e);
ASSERT(shard->usage >= e->totalCharge); // ASSERT(shard->usage >= e->totalCharge);
shard->usage -= e->totalCharge; shard->usage -= e->totalCharge;
lastReference = true; lastReference = true;
} }
@ -532,11 +535,11 @@ static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) {
while (shard->lru.next != &shard->lru) { while (shard->lru.next != &shard->lru) {
SLRUEntry *old = shard->lru.next; SLRUEntry *old = shard->lru.next;
ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); // ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old));
taosLRUCacheShardLRURemove(shard, old); taosLRUCacheShardLRURemove(shard, old);
(void)taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash); (void)taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash);
TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false);
ASSERT(shard->usage >= old->totalCharge); // ASSERT(shard->usage >= old->totalCharge);
shard->usage -= old->totalCharge; shard->usage -= old->totalCharge;
(void)taosArrayPush(lastReferenceList, &old); (void)taosArrayPush(lastReferenceList, &old);
@ -557,7 +560,7 @@ static bool taosLRUCacheShardRef(SLRUCacheShard *shard, LRUHandle *handle) {
SLRUEntry *e = (SLRUEntry *)handle; SLRUEntry *e = (SLRUEntry *)handle;
(void)taosThreadMutexLock(&shard->mutex); (void)taosThreadMutexLock(&shard->mutex);
ASSERT(TAOS_LRU_ENTRY_HAS_REFS(e)); // ASSERT(TAOS_LRU_ENTRY_HAS_REFS(e));
TAOS_LRU_ENTRY_REF(e); TAOS_LRU_ENTRY_REF(e);
(void)taosThreadMutexUnlock(&shard->mutex); (void)taosThreadMutexUnlock(&shard->mutex);
@ -578,7 +581,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b
lastReference = taosLRUEntryUnref(e); lastReference = taosLRUEntryUnref(e);
if (lastReference && TAOS_LRU_ENTRY_IN_CACHE(e)) { if (lastReference && TAOS_LRU_ENTRY_IN_CACHE(e)) {
if (shard->usage > shard->capacity || eraseIfLastRef) { if (shard->usage > shard->capacity || eraseIfLastRef) {
ASSERT(shard->lru.next == &shard->lru || eraseIfLastRef); // ASSERT(shard->lru.next == &shard->lru || eraseIfLastRef);
(void)taosLRUEntryTableRemove(&shard->table, e->keyData, e->keyLength, e->hash); (void)taosLRUEntryTableRemove(&shard->table, e->keyData, e->keyLength, e->hash);
TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); TAOS_LRU_ENTRY_SET_IN_CACHE(e, false);
@ -590,7 +593,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b
} }
if (lastReference && e->value) { if (lastReference && e->value) {
ASSERT(shard->usage >= e->totalCharge); // ASSERT(shard->usage >= e->totalCharge);
shard->usage -= e->totalCharge; shard->usage -= e->totalCharge;
} }
@ -628,7 +631,7 @@ static size_t taosLRUCacheShardGetPinnedUsage(SLRUCacheShard *shard) {
(void)taosThreadMutexLock(&shard->mutex); (void)taosThreadMutexLock(&shard->mutex);
ASSERT(shard->usage >= shard->lruUsage); // ASSERT(shard->usage >= shard->lruUsage);
usage = shard->usage - shard->lruUsage; usage = shard->usage - shard->lruUsage;
(void)taosThreadMutexUnlock(&shard->mutex); (void)taosThreadMutexUnlock(&shard->mutex);
@ -698,8 +701,10 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo
size_t perShard = (capacity + (numShards - 1)) / numShards; size_t perShard = (capacity + (numShards - 1)) / numShards;
for (int i = 0; i < numShards; ++i) { for (int i = 0; i < numShards; ++i) {
if (TSDB_CODE_SUCCESS != if (TSDB_CODE_SUCCESS !=
taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits)) taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits)) {
taosMemoryFree(cache);
return NULL; return NULL;
}
} }
cache->numShards = numShards; cache->numShards = numShards;
@ -718,7 +723,7 @@ void taosLRUCacheCleanup(SLRUCache *cache) {
if (cache) { if (cache) {
if (cache->shards) { if (cache->shards) {
int numShards = cache->numShards; int numShards = cache->numShards;
ASSERT(numShards > 0); // ASSERT(numShards > 0);
for (int i = 0; i < numShards; ++i) { for (int i = 0; i < numShards; ++i) {
taosLRUCacheShardCleanup(&cache->shards[i]); taosLRUCacheShardCleanup(&cache->shards[i]);
} }

50
source/util/src/tpagedbuf.c
View File

@ -2,8 +2,8 @@
#include "tpagedbuf.h" #include "tpagedbuf.h"
#include "taoserror.h" #include "taoserror.h"
#include "tcompression.h" #include "tcompression.h"
#include "tsimplehash.h"
#include "tlog.h" #include "tlog.h"
#include "tsimplehash.h"
#define GET_PAYLOAD_DATA(_p) ((char*)(_p)->pData + POINTER_BYTES) #define GET_PAYLOAD_DATA(_p) ((char*)(_p)->pData + POINTER_BYTES)
#define BUF_PAGE_IN_MEM(_p) ((_p)->pData != NULL) #define BUF_PAGE_IN_MEM(_p) ((_p)->pData != NULL)
@ -27,24 +27,24 @@ struct SPageInfo {
}; };
struct SDiskbasedBuf { struct SDiskbasedBuf {
int32_t numOfPages; int32_t numOfPages;
int64_t totalBufSize; int64_t totalBufSize;
uint64_t fileSize; // disk file size uint64_t fileSize; // disk file size
TdFilePtr pFile; TdFilePtr pFile;
int32_t allocateId; // allocated page id int32_t allocateId; // allocated page id
char* path; // file path char* path; // file path
char* prefix; // file name prefix char* prefix; // file name prefix
int32_t pageSize; // current used page size int32_t pageSize; // current used page size
int32_t inMemPages; // numOfPages that are allocated in memory int32_t inMemPages; // numOfPages that are allocated in memory
SList* freePgList; // free page list SList* freePgList; // free page list
SArray* pIdList; // page id list SArray* pIdList; // page id list
SSHashObj*all; SSHashObj* all;
SList* lruList; SList* lruList;
void* emptyDummyIdList; // dummy id list void* emptyDummyIdList; // dummy id list
void* assistBuf; // assistant buffer for compress/decompress data void* assistBuf; // assistant buffer for compress/decompress data
SArray* pFree; // free area in file SArray* pFree; // free area in file
bool comp; // compressed before flushed to disk bool comp; // compressed before flushed to disk
uint64_t nextPos; // next page flush position uint64_t nextPos; // next page flush position
char* id; // for debug purpose char* id; // for debug purpose
bool printStatis; // Print statistics info when closing this buffer. bool printStatis; // Print statistics info when closing this buffer.
@ -95,7 +95,8 @@ static int32_t doDecompressData(void* data, int32_t srcSize, int32_t* dst, SDisk
} else if (*dst < 0) { } else if (*dst < 0) {
return terrno; return terrno;
} }
return code;; return code;
;
} }
static uint64_t allocateNewPositionInFile(SDiskbasedBuf* pBuf, size_t size) { static uint64_t allocateNewPositionInFile(SDiskbasedBuf* pBuf, size_t size) {
@ -300,7 +301,6 @@ static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) {
SPageInfo* pageInfo = *(SPageInfo**)pn->data; SPageInfo* pageInfo = *(SPageInfo**)pn->data;
SPageInfo* p = *(SPageInfo**)(pageInfo->pData); SPageInfo* p = *(SPageInfo**)(pageInfo->pData);
ASSERT(pageInfo->pageId >= 0 && pageInfo->pn == pn && p == pageInfo);
if (!pageInfo->used) { if (!pageInfo->used) {
break; break;
@ -435,14 +435,14 @@ static char* doExtractPage(SDiskbasedBuf* pBuf, bool* newPage) {
void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) {
pBuf->statis.getPages += 1; pBuf->statis.getPages += 1;
bool newPage = false; bool newPage = false;
char* availablePage = doExtractPage(pBuf, &newPage); char* availablePage = doExtractPage(pBuf, &newPage);
if (availablePage == NULL) { if (availablePage == NULL) {
return NULL; return NULL;
} }
SPageInfo* pi = NULL; SPageInfo* pi = NULL;
int32_t code = 0; int32_t code = 0;
if (listNEles(pBuf->freePgList) != 0) { if (listNEles(pBuf->freePgList) != 0) {
SListNode* pItem = tdListPopHead(pBuf->freePgList); SListNode* pItem = tdListPopHead(pBuf->freePgList);
pi = *(SPageInfo**)pItem->data; pi = *(SPageInfo**)pItem->data;
@ -538,8 +538,6 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
#endif #endif
return (void*)(GET_PAYLOAD_DATA(*pi)); return (void*)(GET_PAYLOAD_DATA(*pi));
} else { // not in memory } else { // not in memory
ASSERT((!BUF_PAGE_IN_MEM(*pi)) && (*pi)->pn == NULL &&
(((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1)));
bool newPage = false; bool newPage = false;
(*pi)->pData = doExtractPage(pBuf, &newPage); (*pi)->pData = doExtractPage(pBuf, &newPage);
@ -700,7 +698,7 @@ void setBufPageDirty(void* pPage, bool dirty) {
void setBufPageCompressOnDisk(SDiskbasedBuf* pBuf, bool comp) { void setBufPageCompressOnDisk(SDiskbasedBuf* pBuf, bool comp) {
pBuf->comp = comp; pBuf->comp = comp;
if (comp && (pBuf->assistBuf == NULL)) { if (comp && (pBuf->assistBuf == NULL)) {
pBuf->assistBuf = taosMemoryMalloc(pBuf->pageSize + 2); // EXTRA BYTES pBuf->assistBuf = taosMemoryMalloc(pBuf->pageSize + 2); // EXTRA BYTES
} }
} }

7
source/util/src/trbtree.c
View File

@ -263,7 +263,6 @@ static void rbtree_delete_fixup(rbtree_t *rbtree, rbnode_t *child, rbnode_t *chi
child_parent->color = BLACK; child_parent->color = BLACK;
return; return;
} }
ASSERTS(sibling != RBTREE_NULL, "sibling is NULL");
/* get a new sibling, by rotating at sibling. See which child /* get a new sibling, by rotating at sibling. See which child
of sibling is red */ of sibling is red */
@ -293,11 +292,9 @@ static void rbtree_delete_fixup(rbtree_t *rbtree, rbnode_t *child, rbnode_t *chi
sibling->color = child_parent->color; sibling->color = child_parent->color;
child_parent->color = BLACK; child_parent->color = BLACK;
if (child_parent->right == child) { if (child_parent->right == child) {
ASSERTS(sibling->left->color == RED, "slibing->left->color=%d not equal RED", sibling->left->color);
sibling->left->color = BLACK; sibling->left->color = BLACK;
rbtree_rotate_right(rbtree, child_parent); rbtree_rotate_right(rbtree, child_parent);
} else { } else {
ASSERTS(sibling->right->color == RED, "slibing->right->color=%d not equal RED", sibling->right->color);
sibling->right->color = BLACK; sibling->right->color = BLACK;
rbtree_rotate_left(rbtree, child_parent); rbtree_rotate_left(rbtree, child_parent);
} }
@ -320,18 +317,15 @@ static void swap_np(rbnode_t **x, rbnode_t **y) {
/** Update parent pointers of child trees of 'parent' */ /** Update parent pointers of child trees of 'parent' */
static void change_parent_ptr(rbtree_t *rbtree, rbnode_t *parent, rbnode_t *old, rbnode_t *new) { static void change_parent_ptr(rbtree_t *rbtree, rbnode_t *parent, rbnode_t *old, rbnode_t *new) {
if (parent == RBTREE_NULL) { if (parent == RBTREE_NULL) {
ASSERTS(rbtree->root == old, "root not equal old");
if (rbtree->root == old) rbtree->root = new; if (rbtree->root == old) rbtree->root = new;
return; return;
} }
ASSERT(parent->left == old || parent->right == old || parent->left == new || parent->right == new);
if (parent->left == old) parent->left = new; if (parent->left == old) parent->left = new;
if (parent->right == old) parent->right = new; if (parent->right == old) parent->right = new;
} }
/** Update parent pointer of a node 'child' */ /** Update parent pointer of a node 'child' */
static void change_child_ptr(rbtree_t *rbtree, rbnode_t *child, rbnode_t *old, rbnode_t *new) { static void change_child_ptr(rbtree_t *rbtree, rbnode_t *child, rbnode_t *old, rbnode_t *new) {
if (child == RBTREE_NULL) return; if (child == RBTREE_NULL) return;
ASSERT(child->parent == old || child->parent == new);
if (child->parent == old) child->parent = new; if (child->parent == old) child->parent = new;
} }
@ -376,7 +370,6 @@ rbnode_t *rbtree_delete(rbtree_t *rbtree, void *key) {
/* now delete to_delete (which is at the location where the smright previously was) */ /* now delete to_delete (which is at the location where the smright previously was) */
} }
ASSERT(to_delete->left == RBTREE_NULL || to_delete->right == RBTREE_NULL);
if (to_delete->left != RBTREE_NULL) if (to_delete->left != RBTREE_NULL)
child = to_delete->left; child = to_delete->left;

3
source/util/src/tsimplehash.c
View File

@ -261,8 +261,7 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, size_t keyLen, cons
static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, size_t keyLen, int32_t index) { static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, size_t keyLen, int32_t index) {
SHNode *pNode = pHashObj->hashList[index]; SHNode *pNode = pHashObj->hashList[index];
while (pNode) { while (pNode) {
const char* p = GET_SHASH_NODE_KEY(pNode, pNode->dataLen); const char *p = GET_SHASH_NODE_KEY(pNode, pNode->dataLen);
ASSERT(keyLen > 0);
if (pNode->keyLen == keyLen && ((*(pHashObj->equalFp))(p, key, keyLen) == 0)) { if (pNode->keyLen == keyLen && ((*(pHashObj->equalFp))(p, key, keyLen) == 0)) {
break; break;

72
source/util/src/tskiplist.c
View File

@ -366,51 +366,6 @@ void *tSkipListDestroyIter(SSkipListIterator *iter) {
return NULL; return NULL;
} }
#ifdef BUILD_NO_CALL
void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) {
return;
}
SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1);
int32_t id = 1;
char *prev = NULL;
while (p != pSkipList->pTail) {
char *key = SL_GET_NODE_KEY(pSkipList, p);
if (prev != NULL) {
ASSERT(pSkipList->comparFn(prev, key) < 0);
}
switch (pSkipList->type) {
case TSDB_DATA_TYPE_INT:
fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key);
break;
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_BIGINT:
fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY:
fprintf(stdout, "%d: %s \n", id++, key);
break;
case TSDB_DATA_TYPE_DOUBLE:
fprintf(stdout, "%d: %lf \n", id++, *(double *)key);
break;
default:
fprintf(stdout, "\n");
}
prev = SL_GET_NODE_KEY(pSkipList, p);
p = SL_NODE_GET_FORWARD_POINTER(p, nlevel - 1);
}
}
#endif
static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward) { static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward) {
for (int32_t i = 0; i < pNode->level; ++i) { for (int32_t i = 0; i < pNode->level; ++i) {
SSkipListNode *x = direction[i]; SSkipListNode *x = direction[i];
@ -538,33 +493,6 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
return hasDupKey; return hasDupKey;
} }
#ifdef BUILD_NO_CALL
static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) {
int32_t level = pNode->level;
uint8_t dupMode = SL_DUP_MODE(pSkipList);
ASSERT(dupMode != SL_DISCARD_DUP_KEY && dupMode != SL_UPDATE_DUP_KEY);
for (int32_t j = level - 1; j >= 0; --j) {
SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j);
SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j);
SL_NODE_GET_FORWARD_POINTER(prev, j) = next;
SL_NODE_GET_BACKWARD_POINTER(next, j) = prev;
}
tSkipListFreeNode(pNode);
pSkipList->size--;
}
// Function must be called after calling tSkipListRemoveNodeImpl() function
static void tSkipListCorrectLevel(SSkipList *pSkipList) {
while (pSkipList->level > 0 &&
SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) {
pSkipList->level -= 1;
}
}
#endif
UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList,
int32_t level) { // record link count in each level int32_t level) { // record link count in each level
#if SKIP_LIST_RECORD_PERFORMANCE #if SKIP_LIST_RECORD_PERFORMANCE

1
source/util/src/tutil.c
View File

@ -124,7 +124,6 @@ char **strsplit(char *z, const char *delim, int32_t *num) {
if (split == NULL) { if (split == NULL) {
return NULL; return NULL;
} }
ASSERTS(NULL != split, "realloc memory failed. size=%d", (int32_t)POINTER_BYTES * size);
} }
} }