xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

change QID log

This commit is contained in:
yihaoDeng 2024-08-22 10:08:26 +08:00
parent 852d8fccd8
commit d3ea7ad6d1
20 changed files with 903 additions and 867 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/examples/c/with_reqid_demo.c
View File

@ -33,7 +33,8 @@ static int DemoWithReqId() {
// connect
TAOS *taos = taos_connect(host, user, password, NULL, port);
if (taos == NULL) {
fprintf(stderr, "Failed to connect to %s:%hu, ErrCode: 0x%x, ErrMessage: %s.\n", host, port, taos_errno(NULL), taos_errstr(NULL));
fprintf(stderr, "Failed to connect to %s:%hu, ErrCode: 0x%x, ErrMessage: %s.\n", host, port, taos_errno(NULL),
taos_errstr(NULL));
taos_cleanup();
return -1;
}
@ -44,7 +45,8 @@ static int DemoWithReqId() {
TAOS_RES *result = taos_query_with_reqid(taos, sql, reqid);
code = taos_errno(result);
if (code != 0) {
fprintf(stderr, "Failed to execute sql with reqId: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code, taos_errstr(result));
fprintf(stderr, "Failed to execute sql with QID: %ld, ErrCode: 0x%x, ErrMessage: %s\n.", reqid, code,
taos_errstr(result));
taos_close(taos);
taos_cleanup();
return -1;
@ -73,6 +75,4 @@ static int DemoWithReqId() {
// ANCHOR_END: with_reqid
}
int main(int argc, char *argv[]) {
return DemoWithReqId();
}
int main(int argc, char *argv[]) { return DemoWithReqId(); }

152
source/client/src/clientEnv.c
View File

@ -26,6 +26,7 @@
#include "qworker.h"
#include "scheduler.h"
#include "tcache.h"
#include "tcompare.h"
#include "tglobal.h"
#include "thttp.h"
#include "tmsg.h"
@ -35,7 +36,6 @@
#include "tsched.h"
#include "ttime.h"
#include "tversion.h"
#include "tcompare.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@ -44,16 +44,16 @@
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
#define ENV_JSON_FALSE_CHECK(c) \
do { \
if (!c) { \
tscError("faild to add item to JSON object");\
code = TSDB_CODE_TSC_FAIL_GENERATE_JSON; \
goto _end; \
} \
#define ENV_JSON_FALSE_CHECK(c) \
do { \
if (!c) { \
tscError("faild to add item to JSON object"); \
code = TSDB_CODE_TSC_FAIL_GENERATE_JSON; \
goto _end; \
} \
} while (0)
#define ENV_ERR_RET(c,info) \
#define ENV_ERR_RET(c, info) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
@ -94,109 +94,110 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) {
int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1);
int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
", current:%d, app current:%d, total:%d, QID:0x%" PRIx64,
pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
}
return code;
}
static void concatStrings(SArray *list, char* buf, int size){
int len = 0;
for(int i = 0; i < taosArrayGetSize(list); i++){
char* db = taosArrayGet(list, i);
static void concatStrings(SArray *list, char *buf, int size) {
int len = 0;
for (int i = 0; i < taosArrayGetSize(list); i++) {
char *db = taosArrayGet(list, i);
if (NULL == db) {
tscError("get dbname failed, buf:%s", buf);
break;
}
char* dot = strchr(db, '.');
char *dot = strchr(db, '.');
if (dot != NULL) {
db = dot + 1;
}
if (i != 0){
if (i != 0) {
(void)strcat(buf, ",");
len += 1;
}
int ret = snprintf(buf + len, size - len, "%s", db);
if (ret < 0) {
if (ret < 0) {
tscError("snprintf failed, buf:%s, ret:%d", buf, ret);
break;
}
len += ret;
if (len >= size){
if (len >= size) {
tscInfo("dbList is truncated, buf:%s, len:%d", buf, len);
break;
}
}
}
static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration){
cJSON* json = cJSON_CreateObject();
static int32_t generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_t reqType, int64_t duration) {
cJSON *json = cJSON_CreateObject();
int32_t code = TSDB_CODE_SUCCESS;
if (json == NULL) {
tscError("[monitor] cJSON_CreateObject failed");
return TSDB_CODE_OUT_OF_MEMORY;
}
char clusterId[32] = {0};
if (snprintf(clusterId, sizeof(clusterId), "%" PRId64, pTscObj->pAppInfo->clusterId) < 0){
if (snprintf(clusterId, sizeof(clusterId), "%" PRId64, pTscObj->pAppInfo->clusterId) < 0) {
tscError("failed to generate clusterId:%" PRId64, pTscObj->pAppInfo->clusterId);
code = TSDB_CODE_FAILED;
goto _end;
}
char startTs[32] = {0};
if (snprintf(startTs, sizeof(startTs), "%" PRId64, pRequest->metric.start/1000) < 0){
tscError("failed to generate startTs:%" PRId64, pRequest->metric.start/1000);
if (snprintf(startTs, sizeof(startTs), "%" PRId64, pRequest->metric.start / 1000) < 0) {
tscError("failed to generate startTs:%" PRId64, pRequest->metric.start / 1000);
code = TSDB_CODE_FAILED;
goto _end;
}
char requestId[32] = {0};
if (snprintf(requestId, sizeof(requestId), "%" PRIu64, pRequest->requestId) < 0){
if (snprintf(requestId, sizeof(requestId), "%" PRIu64, pRequest->requestId) < 0) {
tscError("failed to generate requestId:%" PRIu64, pRequest->requestId);
code = TSDB_CODE_FAILED;
goto _end;
}
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "cluster_id", cJSON_CreateString(clusterId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "start_ts", cJSON_CreateString(startTs)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "request_id", cJSON_CreateString(requestId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "query_time", cJSON_CreateNumber(duration/1000)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "code", cJSON_CreateNumber(pRequest->code)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code))));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows)));
if(pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "cluster_id", cJSON_CreateString(clusterId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "start_ts", cJSON_CreateString(startTs)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "request_id", cJSON_CreateString(requestId)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "query_time", cJSON_CreateNumber(duration / 1000)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "code", cJSON_CreateNumber(pRequest->code)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code))));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(
json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows)));
if (pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen) {
char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen];
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0';
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = tmp;
}else{
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
} else {
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)));
}
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "user", cJSON_CreateString(pTscObj->user)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "user", cJSON_CreateString(pTscObj->user)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_name", cJSON_CreateString(appInfo.appName)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "ip", cJSON_CreateString(tsLocalFqdn)));
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "ip", cJSON_CreateString(tsLocalFqdn)));
char pid[32] = {0};
if (snprintf(pid, sizeof(pid), "%d", appInfo.pid) < 0){
if (snprintf(pid, sizeof(pid), "%d", appInfo.pid) < 0) {
tscError("failed to generate pid:%d", appInfo.pid);
code = TSDB_CODE_FAILED;
goto _end;
}
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_id", cJSON_CreateString(pid)));
if(pRequest->dbList != NULL){
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "process_id", cJSON_CreateString(pid)));
if (pRequest->dbList != NULL) {
char dbList[1024] = {0};
concatStrings(pRequest->dbList, dbList, sizeof(dbList) - 1);
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(dbList)));
}else if(pRequest->pDb != NULL){
} else if (pRequest->pDb != NULL) {
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString(pRequest->pDb)));
}else{
} else {
ENV_JSON_FALSE_CHECK(cJSON_AddItemToObject(json, "db", cJSON_CreateString("")));
}
char* value = cJSON_PrintUnformatted(json);
char *value = cJSON_PrintUnformatted(json);
MonitorSlowLogData data = {0};
data.clusterId = pTscObj->pAppInfo->clusterId;
data.type = SLOW_LOG_WRITE;
@ -212,7 +213,7 @@ _end:
return code;
}
static bool checkSlowLogExceptDb(SRequestObj *pRequest, char* exceptDb) {
static bool checkSlowLogExceptDb(SRequestObj *pRequest, char *exceptDb) {
if (pRequest->pDb != NULL) {
return strcmp(pRequest->pDb, exceptDb) != 0;
}
@ -227,7 +228,7 @@ static bool checkSlowLogExceptDb(SRequestObj *pRequest, char* exceptDb) {
if (dot != NULL) {
db = dot + 1;
}
if(strcmp(db, exceptDb) == 0){
if (strcmp(db, exceptDb) == 0) {
return false;
}
}
@ -248,15 +249,14 @@ static void deregisterRequest(SRequestObj *pRequest) {
int32_t reqType = SLOW_LOG_TYPE_OTHERS;
int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", QID:0x%" PRIx64
" elapsed:%.2f ms, "
"current:%d, app current:%d",
pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst);
if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) {
if ((pRequest->pQuery && pRequest->pQuery->pRoot &&
QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type &&
(0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) ||
if ((pRequest->pQuery && pRequest->pQuery->pRoot && QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type &&
(0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) ||
QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType) {
tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64
"us, planCost:%" PRId64 "us, exec:%" PRId64 "us",
@ -279,7 +279,7 @@ static void deregisterRequest(SRequestObj *pRequest) {
}
}
if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){
if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) {
if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) {
sqlReqLog(pTscObj->id, pRequest->killed, pRequest->code, MONITORSQLTYPEINSERT);
} else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
@ -289,14 +289,15 @@ static void deregisterRequest(SRequestObj *pRequest) {
}
}
if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL || duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) &&
if ((duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThreshold * 1000000UL ||
duration >= pTscObj->pAppInfo->monitorParas.tsSlowLogThresholdTest * 1000000UL) &&
checkSlowLogExceptDb(pRequest, pTscObj->pAppInfo->monitorParas.tsSlowLogExceptDb)) {
(void)atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1);
if (pTscObj->pAppInfo->monitorParas.tsSlowLogScope & reqType) {
taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 " us, Duration:%" PRId64 "us, SQL:%s",
taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration,
pRequest->sqlstr);
if(pTscObj->pAppInfo->monitorParas.tsEnableMonitor){
if (pTscObj->pAppInfo->monitorParas.tsEnableMonitor) {
slowQueryLog(pTscObj->id, pRequest->killed, pRequest->code, duration);
if (TSDB_CODE_SUCCESS != generateWriteSlowLog(pTscObj, pRequest, reqType, duration)) {
tscError("failed to generate write slow log");
@ -389,7 +390,7 @@ void destroyAllRequests(SHashObj *pRequests) {
SRequestObj *pRequest = acquireRequest(*rid);
if (pRequest) {
destroyRequest(pRequest);
(void)releaseRequest(*rid); // ignore error
(void)releaseRequest(*rid); // ignore error
}
pIter = taosHashIterate(pRequests, pIter);
@ -404,7 +405,7 @@ void stopAllRequests(SHashObj *pRequests) {
SRequestObj *pRequest = acquireRequest(*rid);
if (pRequest) {
taos_stop_query(pRequest);
(void)releaseRequest(*rid); // ignore error
(void)releaseRequest(*rid); // ignore error
}
pIter = taosHashIterate(pRequests, pIter);
@ -412,7 +413,7 @@ void stopAllRequests(SHashObj *pRequests) {
}
void destroyAppInst(void *info) {
SAppInstInfo* pAppInfo = *(SAppInstInfo**)info;
SAppInstInfo *pAppInfo = *(SAppInstInfo **)info;
tscDebug("destroy app inst mgr %p", pAppInfo);
int32_t code = taosThreadMutexLock(&appInfo.mutex);
@ -597,28 +598,28 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri
int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
/// return the most previous req ref id
int64_t removeFromMostPrevReq(SRequestObj* pRequest) {
int64_t mostPrevReqRefId = pRequest->self;
SRequestObj* pTmp = pRequest;
int64_t removeFromMostPrevReq(SRequestObj *pRequest) {
int64_t mostPrevReqRefId = pRequest->self;
SRequestObj *pTmp = pRequest;
while (pTmp->relation.prevRefId) {
pTmp = acquireRequest(pTmp->relation.prevRefId);
if (pTmp) {
mostPrevReqRefId = pTmp->self;
(void)releaseRequest(mostPrevReqRefId); // ignore error
(void)releaseRequest(mostPrevReqRefId); // ignore error
} else {
break;
}
}
(void)removeRequest(mostPrevReqRefId); // ignore error
(void)removeRequest(mostPrevReqRefId); // ignore error
return mostPrevReqRefId;
}
void destroyNextReq(int64_t nextRefId) {
if (nextRefId) {
SRequestObj* pObj = acquireRequest(nextRefId);
SRequestObj *pObj = acquireRequest(nextRefId);
if (pObj) {
(void)releaseRequest(nextRefId); // ignore error
(void)releaseRequest(nextRefId); // ignore error
(void)releaseRequest(nextRefId); // ignore error
(void)releaseRequest(nextRefId); // ignore error
}
}
}
@ -638,7 +639,7 @@ void destroySubRequests(SRequestObj *pRequest) {
pTmp = acquireRequest(tmpRefId);
if (pTmp) {
pReqList[++reqIdx] = pTmp;
(void)releaseRequest(tmpRefId); // ignore error
(void)releaseRequest(tmpRefId); // ignore error
} else {
tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId);
break;
@ -646,7 +647,7 @@ void destroySubRequests(SRequestObj *pRequest) {
}
for (int32_t i = reqIdx; i >= 0; i--) {
(void)removeRequest(pReqList[i]->self); // ignore error
(void)removeRequest(pReqList[i]->self); // ignore error
}
tmpRefId = pRequest->relation.nextRefId;
@ -654,8 +655,8 @@ void destroySubRequests(SRequestObj *pRequest) {
pTmp = acquireRequest(tmpRefId);
if (pTmp) {
tmpRefId = pTmp->relation.nextRefId;
(void)removeRequest(pTmp->self); // ignore error
(void)releaseRequest(pTmp->self); // ignore error
(void)removeRequest(pTmp->self); // ignore error
(void)releaseRequest(pTmp->self); // ignore error
} else {
tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId);
break;
@ -758,7 +759,7 @@ void stopAllQueries(SRequestObj *pRequest) {
for (int32_t i = reqIdx; i >= 0; i--) {
taosStopQueryImpl(pReqList[i]);
(void)releaseRequest(pReqList[i]->self); // ignore error
(void)releaseRequest(pReqList[i]->self); // ignore error
}
taosStopQueryImpl(pRequest);
@ -769,7 +770,7 @@ void stopAllQueries(SRequestObj *pRequest) {
if (pTmp) {
tmpRefId = pTmp->relation.nextRefId;
taosStopQueryImpl(pTmp);
(void)releaseRequest(pTmp->self); // ignore error
(void)releaseRequest(pTmp->self); // ignore error
} else {
tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId);
break;
@ -929,7 +930,8 @@ void taos_init_imp(void) {
appInfo.pid = taosGetPId();
appInfo.startTime = taosGetTimestampMs();
appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
appInfo.pInstMapByClusterId = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
appInfo.pInstMapByClusterId =
taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
if (NULL == appInfo.pInstMap || NULL == appInfo.pInstMapByClusterId) {
tscError("failed to allocate memory when init appInfo");
tscInitRes = TSDB_CODE_OUT_OF_MEMORY;
@ -993,8 +995,8 @@ int taos_init() {
return tscInitRes;
}
const char* getCfgName(TSDB_OPTION option) {
const char* name = NULL;
const char *getCfgName(TSDB_OPTION option) {
const char *name = NULL;
switch (option) {
case TSDB_OPTION_SHELL_ACTIVITY_TIMER:
@ -1025,12 +1027,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
char newstr[PATH_MAX];
int len = strlen(str);
if (len > 1 && str[0] != '"' && str[0] != '\'') {
if (len + 2 >= PATH_MAX) {
if (len + 2 >= PATH_MAX) {
tscError("Too long path %s", str);
return -1;
}
newstr[0] = '"';
(void)memcpy(newstr+1, str, len);
(void)memcpy(newstr + 1, str, len);
newstr[len + 1] = '"';
newstr[len + 2] = '\0';
str = newstr;

100
source/client/src/clientImpl.c
View File

@ -15,8 +15,8 @@
#include "cJSON.h"
#include "clientInt.h"
#include "clientMonitor.h"
#include "clientLog.h"
#include "clientMonitor.h"
#include "command.h"
#include "scheduler.h"
#include "tdatablock.h"
@ -28,7 +28,7 @@
#include "tref.h"
#include "tsched.h"
#include "tversion.h"
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static int32_t buildConnectMsg(SRequestObj* pRequest, SMsgSendInfo** pMsgSendInfo);
static bool stringLengthCheck(const char* str, size_t maxsize) {
@ -68,15 +68,13 @@ bool chkRequestKilled(void* param) {
return killed;
}
void cleanupAppInfo() {
taosHashCleanup(appInfo.pInstMap);
}
void cleanupAppInfo() { taosHashCleanup(appInfo.pInstMap); }
static int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj);
int32_t taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port, int connType, STscObj** pObj) {
uint16_t port, int connType, STscObj** pObj) {
TSC_ERR_RET(taos_init());
if (!validateUserName(user)) {
TSC_ERR_RET(TSDB_CODE_TSC_INVALID_USER_LENGTH);
@ -126,7 +124,7 @@ int32_t taos_connect_internal(const char* ip, const char* user, const char* pass
}
SAppInstInfo** pInst = NULL;
int32_t code = taosThreadMutexLock(&appInfo.mutex);
int32_t code = taosThreadMutexLock(&appInfo.mutex);
if (TSDB_CODE_SUCCESS != code) {
tscError("failed to lock app info, code:%s", tstrerror(TAOS_SYSTEM_ERROR(code)));
TSC_ERR_RET(code);
@ -187,14 +185,14 @@ _return:
return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType, pObj);
}
//SAppInstInfo* getAppInstInfo(const char* clusterKey) {
// SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
// if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
// return *ppAppInstInfo;
// } else {
// return NULL;
// }
//}
// SAppInstInfo* getAppInstInfo(const char* clusterKey) {
// SAppInstInfo** ppAppInstInfo = taosHashGet(appInfo.pInstMap, clusterKey, strlen(clusterKey));
// if (ppAppInstInfo != NULL && *ppAppInstInfo != NULL) {
// return *ppAppInstInfo;
// } else {
// return NULL;
// }
// }
void freeQueryParam(SSyncQueryParam* param) {
if (param == NULL) return;
@ -230,7 +228,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
sizeof((*pRequest)->self));
if (err) {
tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
tscError("%" PRId64 " failed to add to request container, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
destroyRequest(*pRequest);
*pRequest = NULL;
@ -241,7 +239,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) {
if (TSDB_CODE_SUCCESS !=
nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
tscError("%" PRId64 " failed to create node allocator, QID:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
destroyRequest(*pRequest);
*pRequest = NULL;
@ -249,7 +247,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
}
}
tscDebugL("0x%" PRIx64 " SQL: %s, reqId:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId);
tscDebugL("0x%" PRIx64 " SQL: %s, QID:0x%" PRIx64, (*pRequest)->self, (*pRequest)->sqlstr, (*pRequest)->requestId);
return TSDB_CODE_SUCCESS;
}
@ -363,10 +361,10 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
} else {
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64,
pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
pRequest->requestId);
}
@ -432,7 +430,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
return TSDB_CODE_SUCCESS;
}
int32_t qnodeRequired(SRequestObj* pRequest, bool *required) {
int32_t qnodeRequired(SRequestObj* pRequest, bool* required) {
if (QUERY_POLICY_VNODE == tsQueryPolicy || QUERY_POLICY_CLIENT == tsQueryPolicy) {
*required = false;
return TSDB_CODE_SUCCESS;
@ -554,7 +552,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr
if (NULL == nodeList) {
return terrno;
}
char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client";
char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client";
int32_t dbNum = taosArrayGetSize(pDbVgList);
for (int32_t i = 0; i < dbNum; ++i) {
@ -568,7 +566,7 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr
}
for (int32_t j = 0; j < vgNum; ++j) {
SVgroupInfo* pInfo = taosArrayGet(pVg, j);
SVgroupInfo* pInfo = taosArrayGet(pVg, j);
if (NULL == pInfo) {
taosArrayDestroy(nodeList);
return TSDB_CODE_OUT_OF_RANGE;
@ -972,8 +970,8 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
break;
}
default:
tscError("0x%" PRIx64 ", invalid exec result for request type %d, reqId:0x%" PRIx64, pRequest->self,
pRequest->type, pRequest->requestId);
tscError("0x%" PRIx64 ", invalid exec result for request type %d, QID:0x%" PRIx64, pRequest->self, pRequest->type,
pRequest->requestId);
code = TSDB_CODE_APP_ERROR;
}
@ -1017,12 +1015,12 @@ void returnToUser(SRequestObj* pRequest) {
(void)releaseRequest(pRequest->relation.userRefId);
return;
} else {
tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self,
tscError("0x%" PRIx64 ", user ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self,
pRequest->relation.userRefId, pRequest->requestId);
}
}
static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock**pBlock) {
static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock** pBlock) {
int64_t lastTs = 0;
TAOS_FIELD* pResFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
@ -1032,7 +1030,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
return code;
}
for(int32_t i = 0; i < numOfFields; ++i) {
for (int32_t i = 0; i < numOfFields; ++i) {
SColumnInfoData colInfoData = createColumnInfoData(pResFields[i].type, pResFields[i].bytes, i + 1);
code = blockDataAppendColInfo(*pBlock, &colInfoData);
if (TSDB_CODE_SUCCESS != code) {
@ -1054,7 +1052,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
lastTs = ts;
}
for(int32_t j = 0; j < numOfFields; ++j) {
for (int32_t j = 0; j < numOfFields; ++j) {
SColumnInfoData* pColInfoData = taosArrayGet((*pBlock)->pDataBlock, j);
code = colDataSetVal(pColInfoData, i, pRow[j], false);
if (TSDB_CODE_SUCCESS != code) {
@ -1069,7 +1067,7 @@ static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock*
(*pBlock)->info.window.ekey = lastTs;
(*pBlock)->info.rows = numOfRows;
tscDebug("lastKey:%"PRId64" numOfRows:%d from all vgroups", lastTs, numOfRows);
tscDebug("lastKey:%" PRId64 " numOfRows:%d from all vgroups", lastTs, numOfRows);
return TSDB_CODE_SUCCESS;
}
@ -1082,7 +1080,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
SSDataBlock* pBlock = NULL;
if (TSDB_CODE_SUCCESS != createResultBlock(res, rowNum, &pBlock)) {
tscError("0x%" PRIx64 ", create result block failed, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId);
tscError("0x%" PRIx64 ", create result block failed, QID:0x%" PRIx64, pRequest->self, pRequest->requestId);
return;
}
@ -1091,7 +1089,7 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
continuePostSubQuery(pNextReq, pBlock);
(void)releaseRequest(pRequest->relation.nextRefId);
} else {
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self,
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self,
pRequest->relation.nextRefId, pRequest->requestId);
}
@ -1110,7 +1108,7 @@ void handlePostSubQuery(SSqlCallbackWrapper* pWrapper) {
continuePostSubQuery(pNextReq, NULL);
(void)releaseRequest(pRequest->relation.nextRefId);
} else {
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self,
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, QID:0x%" PRIx64, pRequest->self,
pRequest->relation.nextRefId, pRequest->requestId);
}
}
@ -1141,11 +1139,11 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
}
taosMemoryFree(pResult);
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self,
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, QID:0x%" PRIx64, pRequest->self,
tstrerror(code), pRequest->retry, pRequest->requestId);
(void)removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
restartAsyncQuery(pRequest, code);
@ -1215,7 +1213,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
}
break;
case QUERY_EXEC_MODE_SCHEDULE: {
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
if (NULL == pMnodeList) {
code = terrno;
break;
@ -1533,7 +1531,7 @@ int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* p
}
int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj) {
SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj) {
*pTscObj = NULL;
int32_t code = createTscObj(user, auth, db, connType, pAppInfo, pTscObj);
if (TSDB_CODE_SUCCESS != code) {
@ -1560,7 +1558,8 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta
}
int64_t transporterId = 0;
code = asyncSendMsgToServer((*pTscObj)->pAppInfo->pTransporter, &(*pTscObj)->pAppInfo->mgmtEp.epSet, &transporterId, body);
code = asyncSendMsgToServer((*pTscObj)->pAppInfo->pTransporter, &(*pTscObj)->pAppInfo->mgmtEp.epSet, &transporterId,
body);
if (TSDB_CODE_SUCCESS != code) {
destroyTscObj(*pTscObj);
tscError("failed to send connect msg to server, code:%s", tstrerror(code));
@ -1577,7 +1576,7 @@ int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __ta
*pTscObj = NULL;
return terrno;
} else {
tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, (*pTscObj)->id,
tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, QID:0x%" PRIx64, (*pTscObj)->id,
(*pTscObj)->connId, (*pTscObj)->pAppInfo->pTransporter, pRequest->requestId);
destroyRequest(pRequest);
}
@ -1708,7 +1707,7 @@ int32_t doProcessMsgFromServer(void* param) {
char tbuf[40] = {0};
TRACE_TO_STR(trace, tbuf);
tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, gtid: %s", pMsg->info.handle,
tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, QID: %s", pMsg->info.handle,
TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf);
if (pSendInfo->requestObjRefId != 0) {
@ -1820,7 +1819,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
}
STscObj* pObj = NULL;
int32_t code = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY, &pObj);
int32_t code = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY, &pObj);
if (TSDB_CODE_SUCCESS == code) {
int64_t* rid = taosMemoryCalloc(1, sizeof(int64_t));
if (NULL == rid) {
@ -1890,7 +1889,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
}
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
SSchedulerReq req = { .syncReq = true, .pFetchRes = (void**)&pResInfo->pData };
SSchedulerReq req = {.syncReq = true, .pFetchRes = (void**)&pResInfo->pData};
pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req);
if (pRequest->code != TSDB_CODE_SUCCESS) {
@ -1905,7 +1904,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
return NULL;
}
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64,
pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId);
STscObj* pTscObj = pRequest->pTscObj;
@ -2033,7 +2032,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) {
}
static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
char* p = (char*)pResultInfo->pData;
char* p = (char*)pResultInfo->pData;
int32_t blockVersion = *(int32_t*)p;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column
@ -2298,7 +2297,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
char* pStart = p;
for (int32_t i = 0; i < numOfCols; ++i) {
if(blockVersion == BLOCK_VERSION_1){
if (blockVersion == BLOCK_VERSION_1) {
colLength[i] = htonl(colLength[i]);
}
if (colLength[i] >= dataLen) {
@ -2733,7 +2732,8 @@ void syncQueryFn(void* param, void* res, int32_t code) {
(void)tsem_post(&pParam->sem);
}
void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) {
void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly,
int8_t source) {
if (sql == NULL || NULL == fp) {
terrno = TSDB_CODE_INVALID_PARA;
if (fp) {
@ -2854,7 +2854,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code,
tstrerror(code), pRequest->requestId);
pResultInfo->pData = pResult;
@ -2877,10 +2877,10 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4);
if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0;
tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code),
tscError("0x%" PRIx64 " fetch results failed, code:%s, QID:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code),
pRequest->requestId);
} else {
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, QID:0x%" PRIx64,
pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
pRequest->requestId);
@ -2932,7 +2932,7 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param
int32_t code = schedulerFetchRows(pRequest->body.queryJob, &req);
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " failed to schedule fetch rows", pRequest->requestId);
pRequest->body.fetchFp(param, pRequest, code);
pRequest->body.fetchFp(param, pRequest, code);
}
}

97
source/client/src/clientMain.c
View File

@ -16,19 +16,19 @@
#include "catalog.h"
#include "clientInt.h"
#include "clientLog.h"
#include "clientStmt.h"
#include "clientMonitor.h"
#include "clientStmt.h"
#include "functionMgt.h"
#include "os.h"
#include "query.h"
#include "scheduler.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tmsg.h"
#include "tref.h"
#include "trpc.h"
#include "version.h"
#include "tcompare.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
@ -120,7 +120,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
}
STscObj *pObj = NULL;
int32_t code = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY, &pObj);
int32_t code = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY, &pObj);
if (TSDB_CODE_SUCCESS == code) {
int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t));
if (NULL == rid) {
@ -183,15 +183,15 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type)
return 0;
}
typedef struct SFetchWhiteListInfo{
int64_t connId;
typedef struct SFetchWhiteListInfo {
int64_t connId;
__taos_async_whitelist_fn_t userCbFn;
void* userParam;
void *userParam;
} SFetchWhiteListInfo;
int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
SFetchWhiteListInfo* pInfo = (SFetchWhiteListInfo*)param;
TAOS* taos = &pInfo->connId;
int32_t fetchWhiteListCallbackFn(void *param, SDataBuf *pMsg, int32_t code) {
SFetchWhiteListInfo *pInfo = (SFetchWhiteListInfo *)param;
TAOS *taos = &pInfo->connId;
if (code != TSDB_CODE_SUCCESS) {
pInfo->userCbFn(pInfo->userParam, code, taos, 0, NULL);
taosMemoryFree(pMsg->pData);
@ -209,7 +209,7 @@ int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
return terrno;
}
uint64_t* pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t));
uint64_t *pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t));
if (pWhiteLists == NULL) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
@ -238,7 +238,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
return;
}
int64_t connId = *(int64_t*)taos;
int64_t connId = *(int64_t *)taos;
STscObj *pTsc = acquireTscObj(connId);
if (NULL == pTsc) {
@ -255,7 +255,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
return;
}
void* pReq = taosMemoryMalloc(msgLen);
void *pReq = taosMemoryMalloc(msgLen);
if (pReq == NULL) {
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
releaseTscObj(connId);
@ -269,7 +269,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
return;
}
SFetchWhiteListInfo* pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo));
SFetchWhiteListInfo *pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo));
if (pParam == NULL) {
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
taosMemoryFree(pReq);
@ -280,9 +280,9 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
pParam->connId = connId;
pParam->userCbFn = fp;
pParam->userParam = param;
SMsgSendInfo* pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
SMsgSendInfo *pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (pSendInfo == NULL) {
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
taosMemoryFree(pParam);
taosMemoryFree(pReq);
releaseTscObj(connId);
@ -297,7 +297,7 @@ void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *pa
pSendInfo->msgType = TDMT_MND_GET_USER_WHITELIST;
int64_t transportId = 0;
SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp);
SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp);
if (TSDB_CODE_SUCCESS != asyncSendMsgToServer(pTsc->pAppInfo->pTransporter, &epSet, &transportId, pSendInfo)) {
tscWarn("failed to async send msg to server");
}
@ -443,7 +443,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
return NULL;
}
if(pRequest->inCallback) {
if (pRequest->inCallback) {
tscError("can not call taos_fetch_row before query callback ends.");
terrno = TSDB_CODE_TSC_INVALID_OPERATION;
return NULL;
@ -454,7 +454,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
SMqRspObj *msg = ((SMqRspObj *)res);
SReqResultInfo *pResultInfo = NULL;
if (msg->common.resIter == -1) {
if(tmqGetNextResInfo(res, true, &pResultInfo) != 0){
if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) {
return NULL;
}
} else {
@ -466,7 +466,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
pResultInfo->current += 1;
return pResultInfo->row;
} else {
if (tmqGetNextResInfo(res, true, &pResultInfo) != 0){
if (tmqGetNextResInfo(res, true, &pResultInfo) != 0) {
return NULL;
}
@ -540,22 +540,23 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
len += sprintf(str + len, "%lf", dv);
} break;
case TSDB_DATA_TYPE_VARBINARY:{
void* data = NULL;
case TSDB_DATA_TYPE_VARBINARY: {
void *data = NULL;
uint32_t size = 0;
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if(taosAscii2Hex(row[i], charLen, &data, &size) < 0){
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if (taosAscii2Hex(row[i], charLen, &data, &size) < 0) {
break;
}
(void)memcpy(str + len, data, size);
len += size;
taosMemoryFree(data);
}break;
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY ||
fields[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (charLen > fields[i].bytes || charLen < 0) {
tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes);
break;
@ -664,7 +665,8 @@ const char *taos_get_client_info() { return version; }
// return int32_t
int taos_affected_rows(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@ -675,7 +677,8 @@ int taos_affected_rows(TAOS_RES *res) {
// return int64_t
int64_t taos_affected_rows64(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return 0;
}
@ -725,7 +728,8 @@ int taos_select_db(TAOS *taos, const char *db) {
}
void taos_stop_query(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) ||
TD_RES_TMQ_BATCH_META(res)) {
return;
}
@ -784,7 +788,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
return pRequest->code;
} else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SReqResultInfo *pResultInfo = NULL;
int32_t code = tmqGetNextResInfo(res, true, &pResultInfo);
int32_t code = tmqGetNextResInfo(res, true, &pResultInfo);
if (code != 0) return code;
pResultInfo->current = pResultInfo->numOfRows;
@ -807,7 +811,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SReqResultInfo *pResultInfo = NULL;
int32_t code = tmqGetNextResInfo(res, false, &pResultInfo);
int32_t code = tmqGetNextResInfo(res, false, &pResultInfo);
if (code != 0) {
(*numOfRows) = 0;
return 0;
@ -937,7 +941,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
SRequestObj *pRequest = pWrapper->pRequest;
SQuery *pQuery = pRequest->pQuery;
qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId);
qDebug("0x%" PRIx64 " start to semantic analysis, QID:0x%" PRIx64, pRequest->self, pRequest->requestId);
int64_t analyseStart = taosGetTimestampUs();
pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart;
@ -953,7 +957,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
(void)memcpy(&pRequest->parseMeta, pResultMeta, sizeof(*pResultMeta));
(void)memset(pResultMeta, 0, sizeof(*pResultMeta));
}
handleQueryAnslyseRes(pWrapper, pResultMeta, code);
}
@ -999,7 +1003,7 @@ void handleSubQueryFromAnalyse(SSqlCallbackWrapper *pWrapper, SMetaData *pResult
}
pNewRequest->pQuery = NULL;
code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pNewRequest->pQuery);
code = nodesMakeNode(QUERY_NODE_QUERY, (SNode **)&pNewRequest->pQuery);
if (pNewRequest->pQuery) {
pNewRequest->pQuery->pRoot = pRoot;
pRoot = NULL;
@ -1056,15 +1060,15 @@ void handleQueryAnslyseRes(SSqlCallbackWrapper *pWrapper, SMetaData *pResultMeta
pRequest->pQuery = NULL;
if (NEED_CLIENT_HANDLE_ERROR(code)) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64,
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId);
restartAsyncQuery(pRequest, code);
return;
}
// return to app directly
tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:0x%" PRIx64, pRequest->self,
tstrerror(code), pRequest->requestId);
tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, QID:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
pRequest->code = code;
returnToUser(pRequest);
}
@ -1113,7 +1117,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
SQuery *pQuery = pRequest->pQuery;
pRequest->metric.ctgCostUs += taosGetTimestampUs() - pRequest->metric.ctgStart;
qDebug("0x%" PRIx64 " start to continue parse, reqId:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId,
qDebug("0x%" PRIx64 " start to continue parse, QID:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId,
tstrerror(code));
if (code == TSDB_CODE_SUCCESS) {
@ -1126,7 +1130,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
}
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code,
tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code,
tstrerror(code), pWrapper->pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
pRequest->pWrapper = NULL;
@ -1143,7 +1147,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
}
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code,
tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pWrapper->pRequest->self, code,
tstrerror(code), pWrapper->pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
pRequest->pWrapper = NULL;
@ -1261,7 +1265,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
}
if (TSDB_CODE_SUCCESS != code) {
tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
tscError("0x%" PRIx64 " error happens, code:%d - %s, QID:0x%" PRIx64, pRequest->self, code, tstrerror(code),
pRequest->requestId);
destorySqlCallbackWrapper(pWrapper);
pRequest->pWrapper = NULL;
@ -1269,9 +1273,9 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
pRequest->pQuery = NULL;
if (NEED_CLIENT_HANDLE_ERROR(code)) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64,
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, QID:0x%" PRIx64,
pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId);
(void)refreshMeta(pRequest->pTscObj, pRequest); //ignore return code,try again
(void)refreshMeta(pRequest->pTscObj, pRequest); // ignore return code,try again
pRequest->prevCode = code;
doAsyncQuery(pRequest, true);
return;
@ -1285,7 +1289,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest);
SRequestObj* pUserReq = pRequest;
SRequestObj *pUserReq = pRequest;
(void)acquireRequest(pRequest->self);
while (pUserReq) {
if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) {
@ -1631,7 +1635,6 @@ TAOS_STMT *taos_stmt_init_with_options(TAOS *taos, TAOS_STMT_OPTIONS *options) {
return pStmt;
}
int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
if (stmt == NULL || sql == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
@ -1874,7 +1877,7 @@ int taos_stmt_close(TAOS_STMT *stmt) {
return stmtClose(stmt);
}
int taos_set_conn_mode(TAOS* taos, int mode, int value) {
int taos_set_conn_mode(TAOS *taos, int mode, int value) {
if (taos == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
@ -1897,6 +1900,4 @@ int taos_set_conn_mode(TAOS* taos, int mode, int value) {
return 0;
}
char* getBuildInfo(){
return buildinfo;
}
char *getBuildInfo() { return buildinfo; }

141
source/client/src/clientRawBlockWrite.c
View File

@ -23,32 +23,31 @@
#include "tglobal.h"
#include "tmsgtype.h"
#define RAW_NULL_CHECK(c) \
do { \
if (c == NULL) { \
code = TSDB_CODE_OUT_OF_MEMORY; \
goto end; \
} \
#define RAW_NULL_CHECK(c) \
do { \
if (c == NULL) { \
code = TSDB_CODE_OUT_OF_MEMORY; \
goto end; \
} \
} while (0)
#define RAW_FALSE_CHECK(c) \
do { \
if (!c) { \
code = TSDB_CODE_INVALID_PARA; \
goto end; \
} \
#define RAW_FALSE_CHECK(c) \
do { \
if (!c) { \
code = TSDB_CODE_INVALID_PARA; \
goto end; \
} \
} while (0)
#define RAW_RETURN_CHECK(c) \
do { \
code = c; \
if (code != 0) { \
goto end; \
} \
#define RAW_RETURN_CHECK(c) \
do { \
code = c; \
if (code != 0) { \
goto end; \
} \
} while (0)
#define LOG_ID_TAG "connId:0x%" PRIx64 ",reqId:0x%" PRIx64
#define LOG_ID_TAG "connId:0x%" PRIx64 ",QID:0x%" PRIx64
#define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId
#define TMQ_META_VERSION "1.0"
@ -57,10 +56,10 @@ static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen
static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
int8_t t, SColCmprWrapper* pColCmprRow, cJSON** pJson) {
static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t,
SColCmprWrapper* pColCmprRow, cJSON** pJson) {
int32_t code = TSDB_CODE_SUCCESS;
int8_t buildDefaultCompress = 0;
int8_t buildDefaultCompress = 0;
if (pColCmprRow->nCols <= 0) {
buildDefaultCompress = 1;
}
@ -82,7 +81,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche
cJSON* columns = cJSON_CreateArray();
RAW_NULL_CHECK(columns);
for (int i = 0; i < schemaRow->nCols; i++) {
cJSON* column = cJSON_CreateObject();
cJSON* column = cJSON_CreateObject();
RAW_NULL_CHECK(column);
SSchema* s = schemaRow->pSchema + i;
cJSON* cname = cJSON_CreateString(s->name);
@ -142,7 +141,7 @@ static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sche
cJSON* tags = cJSON_CreateArray();
RAW_NULL_CHECK(tags);
for (int i = 0; schemaTag && i < schemaTag->nCols; i++) {
cJSON* tag = cJSON_CreateObject();
cJSON* tag = cJSON_CreateObject();
RAW_NULL_CHECK(tag);
SSchema* s = schemaTag->pSchema + i;
cJSON* tname = cJSON_CreateString(s->name);
@ -176,7 +175,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
if (encode != 0) {
const char* encodeStr = columnEncodeStr(encode);
RAW_NULL_CHECK(encodeStr);
cJSON* encodeJson = cJSON_CreateString(encodeStr);
cJSON* encodeJson = cJSON_CreateString(encodeStr);
RAW_NULL_CHECK(encodeJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "encode", encodeJson));
return code;
@ -185,7 +184,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
if (compress != 0) {
const char* compressStr = columnCompressStr(compress);
RAW_NULL_CHECK(compressStr);
cJSON* compressJson = cJSON_CreateString(compressStr);
cJSON* compressJson = cJSON_CreateString(compressStr);
RAW_NULL_CHECK(compressJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "compress", compressJson));
return code;
@ -194,7 +193,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) {
if (level != 0) {
const char* levelStr = columnLevelStr(level);
RAW_NULL_CHECK(levelStr);
cJSON* levelJson = cJSON_CreateString(levelStr);
cJSON* levelJson = cJSON_CreateString(levelStr);
RAW_NULL_CHECK(levelJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "level", levelJson));
return code;
@ -235,7 +234,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_ADD_COLUMN: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colType = cJSON_CreateNumber(field->type);
@ -259,7 +258,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: {
SFieldWithOptions* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colType = cJSON_CreateNumber(field->type);
@ -285,7 +284,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_DROP_COLUMN: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
break;
@ -294,7 +293,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colType = cJSON_CreateNumber(field->type);
@ -320,7 +319,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
RAW_NULL_CHECK(oldField);
TAOS_FIELD* newField = taosArrayGet(req.pFields, 1);
RAW_NULL_CHECK(newField);
cJSON* colName = cJSON_CreateString(oldField->name);
cJSON* colName = cJSON_CreateString(oldField->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
cJSON* colNewName = cJSON_CreateString(newField->name);
@ -331,7 +330,7 @@ static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON**
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: {
TAOS_FIELD* field = taosArrayGet(req.pFields, 0);
RAW_NULL_CHECK(field);
cJSON* colName = cJSON_CreateString(field->name);
cJSON* colName = cJSON_CreateString(field->name);
RAW_NULL_CHECK(colName);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "colName", colName));
RAW_RETURN_CHECK(setCompressOption(json, field->bytes));
@ -405,7 +404,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(tagNumJson);
RAW_FALSE_CHECK(cJSON_AddItemToObject(json, "tagNum", tagNumJson));
cJSON* tags = cJSON_CreateArray();
cJSON* tags = cJSON_CreateArray();
RAW_NULL_CHECK(tags);
SArray* pTagVals = NULL;
RAW_RETURN_CHECK(tTagToValArray(pTag, &pTagVals));
@ -416,13 +415,13 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
uError("p->nTag == 0");
goto end;
}
char* pJson = NULL;
char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
cJSON* tag = cJSON_CreateObject();
cJSON* tag = cJSON_CreateObject();
RAW_NULL_CHECK(tag);
STagVal* pTagVal = taosArrayGet(pTagVals, 0);
RAW_NULL_CHECK(pTagVal);
char* ptname = taosArrayGet(tagName, 0);
char* ptname = taosArrayGet(tagName, 0);
RAW_NULL_CHECK(ptname);
cJSON* tname = cJSON_CreateString(ptname);
RAW_NULL_CHECK(tname);
@ -443,7 +442,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(pTagVal);
cJSON* tag = cJSON_CreateObject();
RAW_NULL_CHECK(tag);
char* ptname = taosArrayGet(tagName, i);
char* ptname = taosArrayGet(tagName, i);
RAW_NULL_CHECK(ptname);
cJSON* tname = cJSON_CreateString(ptname);
RAW_NULL_CHECK(tname);
@ -463,7 +462,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
RAW_NULL_CHECK(buf);
if (!buf) goto end;
if(dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) {
if (dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL) != TSDB_CODE_SUCCESS) {
taosMemoryFree(buf);
goto end;
}
@ -489,8 +488,8 @@ end:
static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSON** pJson) {
int32_t code = 0;
char* string = NULL;
cJSON* json = cJSON_CreateObject();
char* string = NULL;
cJSON* json = cJSON_CreateObject();
RAW_NULL_CHECK(json);
cJSON* type = cJSON_CreateString("create");
RAW_NULL_CHECK(type);
@ -534,8 +533,8 @@ static void processCreateTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
if (pCreateReq->type == TSDB_CHILD_TABLE) {
buildCreateCTableJson(req.pReqs, req.nReqs, pJson);
} else if (pCreateReq->type == TSDB_NORMAL_TABLE) {
buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid,
TSDB_NORMAL_TABLE, &pCreateReq->colCmpr, pJson);
buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE,
&pCreateReq->colCmpr, pJson);
}
}
@ -548,7 +547,7 @@ end:
static void processAutoCreateTable(STaosxRsp* rsp, char** string) {
SDecoder* decoder = NULL;
SVCreateTbReq* pCreateReq = NULL;
int32_t code = 0;
int32_t code = 0;
uDebug("auto create table data:%p", rsp);
if (rsp->createTableNum <= 0) {
uError("processAutoCreateTable rsp->createTableNum <= 0");
@ -563,7 +562,7 @@ static void processAutoCreateTable(STaosxRsp* rsp, char** string) {
// loop to create table
for (int32_t iReq = 0; iReq < rsp->createTableNum; iReq++) {
// decode
void** data = taosArrayGet(rsp->createTableReq, iReq);
void** data = taosArrayGet(rsp->createTableReq, iReq);
RAW_NULL_CHECK(data);
int32_t* len = taosArrayGet(rsp->createTableLen, iReq);
RAW_NULL_CHECK(len);
@ -735,7 +734,8 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 3, 1);
}
RAW_NULL_CHECK(buf);
if(dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) != TSDB_CODE_SUCCESS) {
if (dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL) !=
TSDB_CODE_SUCCESS) {
taosMemoryFree(buf);
goto end;
}
@ -823,7 +823,7 @@ static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
// getTbName(req.tableFName);
char sql[256] = {0};
(void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
req.tsColName, req.skey, req.tsColName, req.ekey);
req.tsColName, req.skey, req.tsColName, req.ekey);
json = cJSON_CreateObject();
RAW_NULL_CHECK(json);
@ -865,7 +865,7 @@ static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
RAW_NULL_CHECK(tableNameList);
for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
SVDropTbReq* pDropTbReq = req.pReqs + iReq;
cJSON* tableName = cJSON_CreateString(pDropTbReq->name);
cJSON* tableName = cJSON_CreateString(pDropTbReq->name);
RAW_NULL_CHECK(tableName);
RAW_FALSE_CHECK(cJSON_AddItemToArray(tableNameList, tableName));
}
@ -910,7 +910,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
}
// build create stable
pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SFieldWithOptions));
RAW_NULL_CHECK (pReq.pColumns);
RAW_NULL_CHECK(pReq.pColumns);
for (int32_t i = 0; i < req.schemaRow.nCols; i++) {
SSchema* pSchema = req.schemaRow.pSchema + i;
SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes};
@ -957,7 +957,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
}
pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen);
RAW_NULL_CHECK(pCmdMsg.pMsg);
if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0){
if (tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) {
code = TSDB_CODE_INVALID_PARA;
goto end;
}
@ -968,7 +968,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
pQuery.msgType = pQuery.pCmdMsg->msgType;
pQuery.stableQuery = true;
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); //ignore, because return value is pRequest
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest
if (pRequest->code == TSDB_CODE_SUCCESS) {
SCatalog* pCatalog = NULL;
@ -1021,7 +1021,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
.requestObjRefId = pRequest->self,
.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)};
SName pName = {0};
(void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); // ignore the return value, always return pName
(void)toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name,
&pName); // ignore the return value, always return pName
STableMeta* pTableMeta = NULL;
code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta);
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
@ -1059,19 +1060,18 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
}
pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen);
RAW_NULL_CHECK(pCmdMsg.pMsg);
if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0){
if (tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq) <= 0) {
code = TSDB_CODE_INVALID_PARA;
goto end;
}
SQuery pQuery = {0};
pQuery.execMode = QUERY_EXEC_MODE_RPC;
pQuery.pCmdMsg = &pCmdMsg;
pQuery.msgType = pQuery.pCmdMsg->msgType;
pQuery.stableQuery = true;
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); //ignore, because return value is pRequest
(void)launchQueryImpl(pRequest, &pQuery, true, NULL); // ignore, because return value is pRequest
if (pRequest->code == TSDB_CODE_SUCCESS) {
// ignore the error code
@ -1391,9 +1391,9 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
}
(void)snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
req.tsColName, req.skey, req.tsColName, req.ekey);
req.tsColName, req.skey, req.tsColName, req.ekey);
TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX);
TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX);
RAW_NULL_CHECK(res);
SRequestObj* pRequest = (SRequestObj*)res;
code = pRequest->code;
@ -1500,7 +1500,6 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
if (TSDB_CODE_SUCCESS != code) goto end;
RAW_RETURN_CHECK(rewriteToVnodeModifyOpStmt(pQuery, pArray));
(void)launchQueryImpl(pRequest, pQuery, true, NULL);
pVgData = NULL;
@ -1573,7 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
@ -1632,7 +1632,8 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
@ -1736,7 +1737,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter);
RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
RAW_NULL_CHECK(fields);
for (int i = 0; i < pSW->nCols; i++) {
fields[i].type = pSW->pSchema[i].type;
@ -1845,7 +1846,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
// find schema data info
for (int j = 0; j < rspObj.rsp.createTableNum; j++) {
void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
RAW_NULL_CHECK(dataTmp);
int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j);
RAW_NULL_CHECK(dataTmp);
@ -1897,7 +1898,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter);
RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
if (fields == NULL) {
SET_ERROR_MSG("calloc fields failed");
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1980,10 +1981,10 @@ static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) {
RAW_NULL_CHECK(len);
void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i);
RAW_NULL_CHECK(tmpBuf);
SDecoder metaCoder = {0};
SDecoder metaCoder = {0};
SMqMetaRsp metaRsp = {0};
tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), *len - sizeof(SMqRspHead));
if(tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0 ) {
if (tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0) {
goto end;
}
cJSON* pItem = NULL;
@ -2013,18 +2014,18 @@ char* tmq_get_json_meta(TAOS_RES* res) {
if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res;
char* string = NULL;
char* string = NULL;
processAutoCreateTable(&pMetaDataRspObj->rsp, &string);
return string;
} else if (TD_RES_TMQ_BATCH_META(res)) {
SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res;
char* string = NULL;
char* string = NULL;
processBatchMetaToJson(&pBatchMetaRspObj->rsp, &string);
return string;
}
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
cJSON* pJson = NULL;
cJSON* pJson = NULL;
processSimpleMeta(&pMetaRspObj->metaRsp, &pJson);
char* string = cJSON_PrintUnformatted(pJson);
cJSON_Delete(pJson);
@ -2103,7 +2104,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
uDebug("tmq get raw type meta:%p", raw);
} else if (TD_RES_TMQ(res)) {
SMqRspObj* rspObj = ((SMqRspObj*)res);
int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw);
int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw);
if (code != 0) {
uError("tmq get raw type error:%d", terrno);
return code;

235
source/client/src/clientTmq.c
View File

@ -37,7 +37,7 @@ struct SMqMgmt {
static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once
volatile int32_t tmqInitRes = 0; // initialize rsp code
static struct SMqMgmt tmqMgmt = {0};
static int8_t pollFlag = 0;
static int8_t pollFlag = 0;
typedef struct {
int32_t code;
@ -121,7 +121,7 @@ struct tmq_t {
typedef struct SAskEpInfo {
int32_t code;
tsem2_t sem;
tsem2_t sem;
} SAskEpInfo;
enum {
@ -191,7 +191,7 @@ typedef struct {
} SMqPollRspWrapper;
typedef struct {
tsem2_t rspSem;
tsem2_t rspSem;
int32_t rspErr;
} SMqSubscribeCbParam;
@ -219,12 +219,12 @@ typedef struct SMqVgCommon {
} SMqVgCommon;
typedef struct SMqSeekParam {
tsem2_t sem;
tsem2_t sem;
int32_t code;
} SMqSeekParam;
typedef struct SMqCommittedParam {
tsem2_t sem;
tsem2_t sem;
int32_t code;
SMqVgOffset vgOffset;
} SMqCommittedParam;
@ -242,18 +242,18 @@ typedef struct {
int32_t waitingRspNum;
int32_t code;
tmq_commit_cb* callbackFn;
void* userParam;
void* userParam;
} SMqCommitCbParamSet;
typedef struct {
SMqCommitCbParamSet* params;
char topicName[TSDB_TOPIC_FNAME_LEN];
int32_t vgId;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
int32_t vgId;
int64_t consumerId;
} SMqCommitCbParam;
typedef struct SSyncCommitInfo {
tsem2_t sem;
tsem2_t sem;
int32_t code;
} SSyncCommitInfo;
@ -334,7 +334,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcasecmp(key, "session.timeout.ms") == 0) {
int64_t tmp = taosStr2int64(value);
if (tmp < 6000 || tmp > 1800000){
if (tmp < 6000 || tmp > 1800000) {
return TMQ_CONF_INVALID;
}
conf->sessionTimeoutMs = tmp;
@ -343,7 +343,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcasecmp(key, "heartbeat.interval.ms") == 0) {
int64_t tmp = taosStr2int64(value);
if (tmp < 1000 || tmp >= conf->sessionTimeoutMs){
if (tmp < 1000 || tmp >= conf->sessionTimeoutMs) {
return TMQ_CONF_INVALID;
}
conf->heartBeatIntervalMs = tmp;
@ -352,7 +352,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
if (strcasecmp(key, "max.poll.interval.ms") == 0) {
int64_t tmp = taosStr2int64(value);
if (tmp < 1000 || tmp > INT32_MAX){
if (tmp < 1000 || tmp > INT32_MAX) {
return TMQ_CONF_INVALID;
}
conf->maxPollIntervalMs = tmp;
@ -515,7 +515,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse
SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len);
if(tEncodeMqVgOffset(&encoder, &pOffset) < 0) {
if (tEncodeMqVgOffset(&encoder, &pOffset) < 0) {
tEncoderClear(&encoder);
taosMemoryFree(buf);
return TSDB_CODE_INVALID_PARA;
@ -562,7 +562,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse
return code;
}
static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic **topic) {
static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic** topic) {
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
for (int32_t i = 0; i < numOfTopics; ++i) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
@ -577,8 +577,8 @@ static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic
return TSDB_CODE_TMQ_INVALID_TOPIC;
}
static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam,
int32_t rspNum, SMqCommitCbParamSet** ppParamSet) {
static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, int32_t rspNum,
SMqCommitCbParamSet** ppParamSet) {
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
if (pParamSet == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -595,7 +595,7 @@ static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, voi
static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg) {
SMqClientTopic* pTopic = NULL;
int32_t code = getTopicByName(tmq, pTopicName, &pTopic);
int32_t code = getTopicByName(tmq, pTopicName, &pTopic);
if (code != 0) {
tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName);
return code;
@ -723,7 +723,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
taosRUnLockLatch(&tmq->lock);
goto end;
}
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName,
numOfVgroups);
for (int32_t j = 0; j < numOfVgroups; j++) {
@ -769,7 +769,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
if (pParamSet->waitingRspNum != 1) {
// count down since waiting rsp num init as 1
code = commitRspCountDown(pParamSet, tmq->consumerId, "", 0);
if (code != 0){
if (code != 0) {
tscError("consumer:0x%" PRIx64 " commit rsp count down failed, code:%s", tmq->consumerId, tstrerror(code));
pParamSet = NULL;
goto end;
@ -824,14 +824,14 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
}
int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
if (code != 0){
if (code != 0) {
goto _return;
}
if (pMsg == NULL || param == NULL) {
code = TSDB_CODE_INVALID_PARA;
goto _return;
}
SMqHbRsp rsp = {0};
code = tDeserializeSMqHbRsp(pMsg->pData, pMsg->len, &rsp);
if (code != 0) {
@ -858,7 +858,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
taosWUnLockLatch(&tmq->lock);
(void)taosReleaseRef(tmqMgmt.rsetId, refId);
}
tDestroySMqHbRsp(&rsp);
_return:
@ -881,32 +881,32 @@ void tmqSendHbReq(void* param, void* tmrId) {
req.epoch = tmq->epoch;
req.pollFlag = atomic_load_8(&pollFlag);
req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
if (req.topics == NULL){
if (req.topics == NULL) {
return;
}
taosRLockLatch(&tmq->lock);
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (pTopic == NULL) {
continue;
}
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
TopicOffsetRows* data = taosArrayReserve(req.topics, 1);
if (data == NULL){
if (data == NULL) {
continue;
}
(void)strcpy(data->topicName, pTopic->topicName);
data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows));
if (data->offsetRows == NULL){
if (data->offsetRows == NULL) {
continue;
}
for (int j = 0; j < numOfVgroups; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
if (pVg == NULL){
if (pVg == NULL) {
continue;
}
OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1);
if (offRows == NULL){
OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1);
if (offRows == NULL) {
continue;
}
offRows->vgId = pVg->vgId;
@ -964,7 +964,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
(void)atomic_val_compare_exchange_8(&pollFlag, 1, 0);
OVER:
tDestroySMqHbReq(&req);
if(tmrId != NULL){
if (tmrId != NULL) {
(void)taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer);
}
(void)taosReleaseRef(tmqMgmt.rsetId, refId);
@ -1006,14 +1006,15 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
continue;
}
tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
(void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->epTimer);
(void)taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->epTimer);
} else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn;
asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId,
pTmq->autoCommitInterval / 1000.0);
(void)taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->commitTimer);
&pTmq->commitTimer);
} else {
tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
}
@ -1100,16 +1101,16 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
taosRLockLatch(&tmq->lock);
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i);
if(topic == NULL) {
if (topic == NULL) {
tscError("topic is null");
continue;
}
char* tmp = strchr(topic->topicName, '.');
if(tmp == NULL) {
if (tmp == NULL) {
tscError("topic name is invalid:%s", topic->topicName);
continue;
}
if(tmq_list_append(*topics, tmp+ 1) != 0) {
if (tmq_list_append(*topics, tmp + 1) != 0) {
tscError("failed to append topic:%s", tmp + 1);
continue;
}
@ -1227,27 +1228,31 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
}
code = taosOpenQueue(&pTmq->mqueue);
if (code) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId);
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("open queue failed")
goto _failed;
}
code = taosOpenQueue(&pTmq->delayedTask);
if (code) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId);
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("open delayed task queue failed")
goto _failed;
}
code = taosAllocateQall(&pTmq->qall);
if (code) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId);
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("allocate qall failed")
goto _failed;
}
if (conf->groupId[0] == 0) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code), pTmq->groupId);
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, tstrerror(code),
pTmq->groupId);
SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty")
goto _failed;
}
@ -1287,8 +1292,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
// init semaphore
if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) {
tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, tstrerror(TAOS_SYSTEM_ERROR(errno)),
pTmq->groupId);
tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId,
tstrerror(TAOS_SYSTEM_ERROR(errno)), pTmq->groupId);
SET_ERROR_MSG_TMQ("init t_sem failed")
goto _failed;
}
@ -1371,7 +1376,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
SName name = {0};
code = tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic));
if (code) {
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId, code);
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to set topic name, code:%d", tmq->consumerId, tmq->groupId,
code);
goto FAIL;
}
char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
@ -1382,7 +1388,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
code = tNameExtractFullName(&name, topicFName);
if (code) {
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId, code);
tscError("consumer:0x%" PRIx64 " cgroup:%s, failed to extract topic name, code:%d", tmq->consumerId, tmq->groupId,
code);
taosMemoryFree(topicFName);
goto FAIL;
}
@ -1459,7 +1466,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
}
tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer);
tmq->commitTimer =taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer);
tmq->commitTimer =
taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer);
if (tmq->epTimer == NULL || tmq->commitTimer == NULL) {
code = TSDB_CODE_TSC_INTERNAL_ERROR;
goto FAIL;
@ -1516,12 +1524,12 @@ static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId) {
}
int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tmq_t* tmq = NULL;
tmq_t* tmq = NULL;
SMqPollRspWrapper* pRspWrapper = NULL;
int8_t rspType = 0;
int32_t vgId = 0;
uint64_t requestId = 0;
SMqPollCbParam* pParam = (SMqPollCbParam*)param;
int8_t rspType = 0;
int32_t vgId = 0;
uint64_t requestId = 0;
SMqPollCbParam* pParam = (SMqPollCbParam*)param;
if (pMsg == NULL) {
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
@ -1530,7 +1538,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFreeClear(pMsg->pEpSet);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
int64_t refId = pParam->refId;
int64_t refId = pParam->refId;
vgId = pParam->vgId;
requestId = pParam->requestId;
tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
@ -1562,7 +1570,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (msgEpoch < clientEpoch) {
// do not write into queue since updating epoch reset
tscWarn("consumer:0x%" PRIx64
" msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64,
" msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, QID:0x%" PRIx64,
tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId);
code = TSDB_CODE_TMQ_CONSUMER_MISMATCH;
goto END;
@ -1589,7 +1597,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset);
tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64,
tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, QID:0x%" PRIx64,
tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId);
} else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SDecoder decoder = {0};
@ -1621,23 +1629,24 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
}
tDecoderClear(&decoder);
(void)memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead));
tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, vgId,requestId);
tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, QID:0x%" PRIx64, tmq->consumerId, vgId,
requestId);
} else { // invalid rspType
tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType);
}
END:
if (pRspWrapper){
if (pRspWrapper) {
pRspWrapper->code = code;
pRspWrapper->vgId = vgId;
(void)strcpy(pRspWrapper->topicName, pParam->topicName);
code = taosWriteQitem(tmq->mqueue, pRspWrapper);
if(code != 0){
if (code != 0) {
tscError("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code);
}
}
int32_t total = taosQueueItemSize(tmq->mqueue);
tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64,
tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, QID:0x%" PRIx64,
tmq ? tmq->consumerId : 0, rspType, vgId, total, requestId);
if (tmq) (void)tsem2_post(&tmq->rspSem);
@ -1676,7 +1685,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
}
for (int32_t j = 0; j < vgNumGet; j++) {
SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
if (pVgEp == NULL){
if (pVgEp == NULL) {
continue;
}
(void)sprintf(vgKey, "%s:%d", pTopic->topicName, pVgEp->vgId);
@ -1712,7 +1721,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
clientVg.offsetInfo.committedOffset = offsetNew;
clientVg.offsetInfo.beginOffset = offsetNew;
}
if (taosArrayPush(pTopic->vgs, &clientVg) == NULL){
if (taosArrayPush(pTopic->vgs, &clientVg) == NULL) {
tscError("consumer:0x%" PRIx64 ", failed to push vg:%d into topic:%s", tmq->consumerId, pVgEp->vgId,
pTopic->topicName);
freeClientVg(&clientVg);
@ -1773,7 +1782,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
.commitOffset = pVgCur->offsetInfo.committedOffset,
.numOfRows = pVgCur->numOfRows,
.vgStatus = pVgCur->vgStatus};
if(taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0){
if (taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)) != 0) {
tscError("consumer:0x%" PRIx64 ", failed to put vg:%d into hashmap", tmq->consumerId, pVgCur->vgId);
}
}
@ -1787,7 +1796,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
continue;
}
initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq);
if(taosArrayPush(newTopics, &topic) == NULL){
if (taosArrayPush(newTopics, &topic) == NULL) {
tscError("consumer:0x%" PRIx64 ", failed to push topic:%s into new topics", tmq->consumerId, topic.topicName);
freeClientTopic(&topic);
}
@ -1919,7 +1928,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
pDataRsp->withSchema = true;
pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*));
if (pDataRsp->blockSchema == NULL){
if (pDataRsp->blockSchema == NULL) {
tscError("failed to allocate memory for blockSchema");
return;
}
@ -1938,7 +1947,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
if (schema) {
if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL){
if (taosArrayPush(pDataRsp->blockSchema, &schema) == NULL) {
tscError("failed to push schema into blockSchema");
continue;
}
@ -1947,7 +1956,8 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
}
}
int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj** ppRspObj) {
int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows,
SMqRspObj** ppRspObj) {
SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
if (pRspObj == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -1959,7 +1969,8 @@ int32_t tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, in
return 0;
}
int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqTaosxRspObj** ppRspObj) {
int32_t tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows,
SMqTaosxRspObj** ppRspObj) {
SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
if (pRspObj == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -2030,7 +2041,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
char offsetFormatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset);
code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, reqId:0x%" PRIx64,
tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, QID:0x%" PRIx64,
pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId);
if (code != 0) {
return code;
@ -2056,10 +2067,10 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
for (int i = 0; i < numOfTopics; i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (pTopic == NULL){
if (pTopic == NULL) {
continue;
}
int32_t numOfVg = taosArrayGetSize(pTopic->vgs);
int32_t numOfVg = taosArrayGetSize(pTopic->vgs);
if (pTopic->noPrivilege) {
tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName);
continue;
@ -2069,7 +2080,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
if (pVg == NULL) {
continue;
}
int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs;
int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs;
if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms
tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId,
tmq->epoch, pVg->vgId);
@ -2203,7 +2214,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset);
if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64
", total:%" PRId64 ", reqId:0x%" PRIx64,
", total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId);
pVg->emptyBlockReceiveTs = taosGetTimestampMs();
tmqFreeRspWrapper(pRspWrapper);
@ -2220,13 +2231,13 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
pVg->blockSleepForReplay = pRsp->rsp.sleepTime;
if (pVg->blockSleepForReplay > 0) {
if (taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, (void*)(tmq->refId), tmqMgmt.timer) == NULL) {
tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%"PRId64, tmq->consumerId,
pVg->vgId, pVg->blockSleepForReplay);
tscError("consumer:0x%" PRIx64 " failed to start replay timer, vgId:%d, sleep:%" PRId64,
tmq->consumerId, pVg->vgId, pVg->blockSleepForReplay);
}
}
}
tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId);
taosFreeQitem(pRspWrapper);
@ -2302,7 +2313,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever,
tmq->consumerId, true);
SMqBatchMetaRspObj* pRsp = NULL;
(void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp) ;
(void)tmqBuildBatchMetaRspFromWrapper(pollRspWrapper, &pRsp);
taosFreeQitem(pRspWrapper);
taosWUnLockLatch(&tmq->lock);
return pRsp;
@ -2337,7 +2348,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tmq->consumerId, pDataRsp->blockNum != 0);
if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64,
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId);
pVg->emptyBlockReceiveTs = taosGetTimestampMs();
tmqFreeRspWrapper(pRspWrapper);
@ -2349,9 +2360,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
}
// build rsp
int64_t numOfRows = 0;
SMqTaosxRspObj* pRsp = NULL;
if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) !=0 ) {
int64_t numOfRows = 0;
SMqTaosxRspObj* pRsp = NULL;
if (tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows, &pRsp) != 0) {
tscError("consumer:0x%" PRIx64 " build taosx rsp failed, vgId:%d", tmq->consumerId, pVg->vgId);
}
tmq->totalRows += numOfRows;
@ -2359,7 +2370,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
", vg total:%" PRId64 ", total:%" PRId64 ", QID:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,
pollRspWrapper->reqId);
@ -2471,7 +2482,7 @@ static void displayConsumeStatistics(tmq_t* pTmq) {
tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId);
}
static int32_t innerClose(tmq_t* tmq){
static int32_t innerClose(tmq_t* tmq) {
if (tmq->status != TMQ_CONSUMER_STATUS__READY) {
tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId);
return 0;
@ -2485,7 +2496,7 @@ static int32_t innerClose(tmq_t* tmq){
tmqSendHbReq((void*)(tmq->refId), NULL);
tmq_list_t* lst = tmq_list_new();
if (lst == NULL){
if (lst == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = tmq_subscribe(tmq, lst);
@ -2499,7 +2510,7 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
int32_t code = 0;
if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) {
code = innerClose(tmq);
if(code == 0){
if (code == 0) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED);
}
}
@ -2514,12 +2525,12 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
int32_t code = 0;
if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__CLOSED) {
code = innerClose(tmq);
if(code == 0){
if (code == 0) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__CLOSED);
}
}
if (code == 0){
if (code == 0) {
(void)taosRemoveRef(tmqMgmt.rsetId, tmq->refId);
}
return code;
@ -2562,13 +2573,13 @@ const char* tmq_get_topic_name(TAOS_RES* res) {
return NULL;
}
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
char *tmp = strchr(((SMqRspObjCommon*)res)->topic, '.');
char* tmp = strchr(((SMqRspObjCommon*)res)->topic, '.');
if (tmp == NULL) {
return NULL;
}
return tmp + 1;
} else if (TD_RES_TMQ_META(res)) {
char *tmp = strchr(((SMqMetaRspObj*)res)->topic, '.');
char* tmp = strchr(((SMqMetaRspObj*)res)->topic, '.');
if (tmp == NULL) {
return NULL;
}
@ -2584,13 +2595,13 @@ const char* tmq_get_db_name(TAOS_RES* res) {
}
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) {
char *tmp = strchr(((SMqRspObjCommon*)res)->db, '.');
char* tmp = strchr(((SMqRspObjCommon*)res)->db, '.');
if (tmp == NULL) {
return NULL;
}
return tmp + 1;
} else if (TD_RES_TMQ_META(res)) {
char *tmp = strchr(((SMqMetaRspObj*)res)->db, '.');
char* tmp = strchr(((SMqMetaRspObj*)res)->db, '.');
if (tmp == NULL) {
return NULL;
}
@ -2690,7 +2701,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) {
int32_t code = 0;
SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo));
if(pInfo == NULL) {
if (pInfo == NULL) {
tscError("failed to allocate memory for sync commit");
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -2836,7 +2847,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
if (param == NULL) {
goto FAIL;
}
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
if (tmq == NULL) {
@ -2857,7 +2868,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch);
if (pParam->sync) {
SMqAskEpRsp rsp = {0};
if(tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL){
if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp) != NULL) {
(void)doUpdateLocalEp(tmq, head->epoch, &rsp);
}
tDeleteSMqAskEpRsp(&rsp);
@ -2871,10 +2882,10 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
pWrapper->epoch = head->epoch;
(void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL){
if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL) {
tmqFreeRspWrapper((SMqRspWrapper*)pWrapper);
taosFreeQitem(pWrapper);
}else{
} else {
(void)taosWriteQitem(tmq->mqueue, pWrapper);
}
}
@ -2972,7 +2983,7 @@ int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) {
sendInfo->msgType = TDMT_MND_TMQ_ASK_EP;
SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp);
tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId);
tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, QID:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId);
return asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo);
}
@ -3015,7 +3026,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
if (common->withSchema) {
doFreeReqResultInfo(&pRspObj->resInfo);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter);
if (pSW){
if (pSW) {
TAOS_CHECK_RETURN(setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols));
}
}
@ -3032,9 +3043,9 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes
pRspObj->resInfo.precision = precision;
pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows;
int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, pRspObj->resInfo.numOfRows,
convertUcs4);
if (code != 0){
int32_t code = setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols,
pRspObj->resInfo.numOfRows, convertUcs4);
if (code != 0) {
return code;
}
*pResInfo = &pRspObj->resInfo;
@ -3062,18 +3073,18 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
code = tDecodeMqDataRsp(&decoder, &rsp);
tDecoderClear(&decoder);
if (code != 0){
if (code != 0) {
goto END;
}
SMqRspHead* pHead = pMsg->pData;
SMqRspHead* pHead = pMsg->pData;
tmq_topic_assignment assignment = {.begin = pHead->walsver,
.end = pHead->walever + 1,
.currentOffset = rsp.common.rspOffset.version,
.vgId = pParam->vgId};
(void)taosThreadMutexLock(&pCommon->mutex);
if(taosArrayPush(pCommon->pList, &assignment) == NULL){
if (taosArrayPush(pCommon->pList, &assignment) == NULL) {
tscError("consumer:0x%" PRIx64 " failed to push the wal info from vgId:%d for topic:%s", pCommon->consumerId,
pParam->vgId, pCommon->pTopicName);
code = TSDB_CODE_TSC_INTERNAL_ERROR;
@ -3184,7 +3195,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep
taosMemoryFree(sendInfo);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tsem2_init(&pParam->sem, 0, 0) != 0){
if (tsem2_init(&pParam->sem, 0, 0) != 0) {
taosMemoryFree(buf);
taosMemoryFree(sendInfo);
taosMemoryFree(pParam);
@ -3348,7 +3359,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
taosWLockLatch(&tmq->lock);
SMqClientTopic* pTopic = NULL;
int32_t code = getTopicByName(tmq, tname, &pTopic);
int32_t code = getTopicByName(tmq, tname, &pTopic);
if (code != 0) {
tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName);
goto end;
@ -3358,10 +3369,10 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
*numOfAssignment = taosArrayGetSize(pTopic->vgs);
for (int32_t j = 0; j < (*numOfAssignment); ++j) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j);
if (pClientVg == NULL){
if (pClientVg == NULL) {
continue;
}
int32_t type = pClientVg->offsetInfo.beginOffset.type;
int32_t type = pClientVg->offsetInfo.beginOffset.type;
if (isInSnapshotMode(type, tmq->useSnapshot)) {
tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type);
code = TSDB_CODE_TMQ_SNAPSHOT_ERROR;
@ -3381,7 +3392,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
for (int32_t j = 0; j < (*numOfAssignment); ++j) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j);
if (pClientVg == NULL){
if (pClientVg == NULL) {
continue;
}
if (pClientVg->offsetInfo.beginOffset.type != TMQ_OFFSET__LOG) {
@ -3410,7 +3421,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
if (tsem2_init(&pCommon->rsp, 0, 0) != 0){
if (tsem2_init(&pCommon->rsp, 0, 0) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
@ -3420,7 +3431,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
for (int32_t i = 0; i < (*numOfAssignment); ++i) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i);
if (pClientVg == NULL){
if (pClientVg == NULL) {
continue;
}
SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam));
@ -3479,8 +3490,8 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
char offsetFormatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset);
tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64,
tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, QID:0x%" PRIx64, tmq->consumerId,
pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId);
code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo);
if (code != 0) {
goto end;
@ -3504,7 +3515,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a
for (int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) {
SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i);
if (pClientVg == NULL){
if (pClientVg == NULL) {
continue;
}
if (pClientVg->vgId != p->vgId) {
@ -3631,7 +3642,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_
taosMemoryFree(sendInfo);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tsem2_init(&pParam->sem, 0, 0) != 0){
if (tsem2_init(&pParam->sem, 0, 0) != 0) {
taosMemoryFree(msg);
taosMemoryFree(sendInfo);
taosMemoryFree(pParam);

14
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -83,12 +83,12 @@ extern "C" {
}\
}
#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGInfo(param, ...) {if (dDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGDebug(param, ...) {if (dDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", QID:%s", __VA_ARGS__, buf);}}
#define dGTrace(param, ...) {if (dDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", QID:%s", __VA_ARGS__, buf);}}
// clang-format on
@ -208,7 +208,7 @@ void dmGetMonitorSystemInfo(SMonSysInfo *pInfo);
int32_t dmReadFile(const char *path, const char *name, bool *pDeployed);
int32_t dmWriteFile(const char *path, const char *name, bool deployed);
int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile);
//int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile);
// int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile);
// dmodule.c
int32_t dmInitDndInfo(SDnodeData *pData);

14
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -41,12 +41,12 @@ extern "C" {
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
#define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);}}
#define mGFatal(param, ...) { if (mDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", QID:%s", __VA_ARGS__, buf);}}
#define mGError(param, ...) { if (mDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", QID:%s", __VA_ARGS__, buf);}}
#define mGWarn(param, ...) { if (mDebugFlag & DEBUG_WARN){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", QID:%s", __VA_ARGS__, buf);}}
#define mGInfo(param, ...) { if (mDebugFlag & DEBUG_INFO){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", QID:%s", __VA_ARGS__, buf);}}
#define mGDebug(param, ...) { if (mDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", QID:%s", __VA_ARGS__, buf);}}
#define mGTrace(param, ...) { if (mDebugFlag & DEBUG_TRACE){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", QID:%s", __VA_ARGS__, buf);}}
// clang-format on
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
@ -54,7 +54,7 @@ extern "C" {
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
typedef int32_t (*MndMsgFp)(SRpcMsg *pMsg);
typedef int32_t (*MndMsgFpExt)(SRpcMsg *pMsg, SQueueInfo* pInfo);
typedef int32_t (*MndMsgFpExt)(SRpcMsg *pMsg, SQueueInfo *pInfo);
typedef int32_t (*MndInitFp)(SMnode *pMnode);
typedef void (*MndCleanupFp)(SMnode *pMnode);
typedef int32_t (*ShowRetrieveFp)(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);

12
source/dnode/vnode/src/inc/vnd.h
View File

@ -32,12 +32,12 @@ extern "C" {
#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", DEBUG_DEBUG, vDebugFlag, __VA_ARGS__); }} while(0)
#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", DEBUG_TRACE, vDebugFlag, __VA_ARGS__); }} while(0)
#define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define vGTrace(param, ...) do { if (vDebugFlag & DEBUG_TRACE) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGFatal(param, ...) do { if (vDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGError(param, ...) do { if (vDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
// clang-format on

14
source/dnode/vnode/src/tq/tq.c
View File

@ -173,7 +173,7 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) {
dataRsp.common.blockNum = 0;
char buf[TSDB_OFFSET_LEN] = {0};
(void)tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset);
tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, reqId:0x%" PRIx64, req.consumerId, vgId, buf,
tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, QID:0x%" PRIx64, req.consumerId, vgId, buf,
req.reqId);
code = tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
@ -193,7 +193,7 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq*
(void)tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset);
(void)tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset);
tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64,
tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, QID:0x%" PRIx64,
vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId);
return tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever);
@ -421,7 +421,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
char buf[TSDB_OFFSET_LEN] = {0};
(void)tFormatOffset(buf, TSDB_OFFSET_LEN, &reqOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64,
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, QID:0x%" PRIx64,
consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId);
code = tqExtractDataForMq(pTq, pHandle, &req, pMsg);
@ -1204,13 +1204,13 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
}
if (req.mndTrigger) {
tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ", pTask->id.idStr,
vgId, pTask->info.taskLevel, req.checkpointId, req.transId);
tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d, ",
pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId);
} else {
const char* pPrevStatus = streamTaskGetStatusStr(streamTaskGetPrevStatus(pTask));
tqInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64
", transId:%d after transfer-state, prev status:%s",
pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId, pPrevStatus);
", transId:%d after transfer-state, prev status:%s",
pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId, pPrevStatus);
}
code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask);

145
source/dnode/vnode/src/tq/tqRead.c
View File

@ -74,7 +74,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
pCreateReq = req.pReqs + iReq;
if (pCreateReq->type == TSDB_CHILD_TABLE && pCreateReq->ctb.suid == tbSuid) {
reqNew.nReqs++;
if (taosArrayPush(reqNew.pArray, pCreateReq) == NULL){
if (taosArrayPush(reqNew.pArray, pCreateReq) == NULL) {
taosArrayDestroy(reqNew.pArray);
tDeleteSVCreateTbBatchReq(&req);
goto end;
@ -155,7 +155,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
pDropReq = req.pReqs + iReq;
if (pDropReq->suid == tbSuid) {
reqNew.nReqs++;
if (taosArrayPush(reqNew.pArray, pDropReq) == NULL){
if (taosArrayPush(reqNew.pArray, pDropReq) == NULL) {
taosArrayDestroy(reqNew.pArray);
goto end;
}
@ -214,12 +214,12 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t
while (offset <= appliedVer) {
if (walFetchHead(pHandle->pWalReader, offset) < 0) {
tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64
", no more log to return, reqId:0x%" PRIx64 " 0x%" PRIx64,
", no more log to return, QID:0x%" PRIx64 " 0x%" PRIx64,
pHandle->consumerId, pHandle->epoch, vgId, offset, reqId, id);
goto END;
}
tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64 " 0x%" PRIx64,
tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, QID:0x%" PRIx64 " 0x%" PRIx64,
vgId, pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id);
if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) {
@ -261,10 +261,10 @@ END:
bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; }
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid){
bool ret = false;
SSchemaWrapper *schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
bool ret = false;
SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true;
}
tDeleteSchemaWrapper(schema);
@ -486,7 +486,7 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) {
(pReader->nextBlk + 1), numOfBlocks, idstr);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if (pSubmitTbData == NULL){
if (pSubmitTbData == NULL) {
return false;
}
if (pReader->tbIdHash == NULL) {
@ -639,9 +639,9 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol
int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) {
tqTrace("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk);
int32_t code = 0;
int32_t line = 0;
STSchema* pTSchema = NULL;
int32_t code = 0;
int32_t line = 0;
STSchema* pTSchema = NULL;
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++);
TSDB_CHECK_NULL(pSubmitTbData, code, line, END, terrno);
SSDataBlock* pBlock = pReader->pResBlock;
@ -712,10 +712,11 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
continue;
}
SColData* pCol = taosArrayGet(pCols, sourceIdx);
SColData* pCol = taosArrayGet(pCols, sourceIdx);
TSDB_CHECK_NULL(pCol, code, line, END, terrno);
SColVal colVal = {0};
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual, sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
SColVal colVal = {0};
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual,
sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
if (pCol->cid < pColData->info.colId) {
sourceIdx++;
} else if (pCol->cid == pColData->info.colId) {
@ -738,7 +739,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
TSDB_CHECK_NULL(pTSchema, code, line, END, terrno);
for (int32_t i = 0; i < numOfRows; i++) {
SRow* pRow = taosArrayGetP(pRows, i);
SRow* pRow = taosArrayGetP(pRows, i);
TSDB_CHECK_NULL(pRow, code, line, END, terrno);
int32_t sourceIdx = 0;
for (int32_t j = 0; j < colActual; j++) {
@ -765,44 +766,43 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
}
}
}
}
END:
if (code != 0){
if (code != 0) {
tqError("tqRetrieveDataBlock failed, line:%d, code:%d", line, code);
}
taosMemoryFreeClear(pTSchema);
return code;
}
#define PROCESS_VAL \
if (curRow == 0) {\
assigned[j] = !COL_VAL_IS_NONE(&colVal);\
buildNew = true;\
} else {\
bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal);\
if (currentRowAssigned != assigned[j]) {\
assigned[j] = currentRowAssigned;\
buildNew = true;\
}\
#define PROCESS_VAL \
if (curRow == 0) { \
assigned[j] = !COL_VAL_IS_NONE(&colVal); \
buildNew = true; \
} else { \
bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal); \
if (currentRowAssigned != assigned[j]) { \
assigned[j] = currentRowAssigned; \
buildNew = true; \
} \
}
#define SET_DATA \
if (colVal.cid < pColData->info.colId) {\
sourceIdx++;\
} else if (colVal.cid == pColData->info.colId) {\
TQ_ERR_GO_TO_END(doSetVal(pColData, curRow - lastRow, &colVal));\
sourceIdx++;\
targetIdx++;\
#define SET_DATA \
if (colVal.cid < pColData->info.colId) { \
sourceIdx++; \
} else if (colVal.cid == pColData->info.colId) { \
TQ_ERR_GO_TO_END(doSetVal(pColData, curRow - lastRow, &colVal)); \
sourceIdx++; \
targetIdx++; \
}
static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks,
SArray* schemas, SSchemaWrapper* pSchemaWrapper, char* assigned,
int32_t numOfRows, int32_t curRow, int32_t* lastRow){
int32_t code = 0;
SSchemaWrapper* pSW = NULL;
SSDataBlock* block = NULL;
static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas,
SSchemaWrapper* pSchemaWrapper, char* assigned, int32_t numOfRows, int32_t curRow,
int32_t* lastRow) {
int32_t code = 0;
SSchemaWrapper* pSW = NULL;
SSDataBlock* block = NULL;
if (taosArrayGetSize(blocks) > 0) {
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
TQ_NULL_GO_TO_END(pLastBlock);
@ -834,33 +834,34 @@ END:
taosMemoryFree(block);
return code;
}
static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas){
int32_t code = 0;
int32_t curRow = 0;
int32_t lastRow = 0;
static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) {
int32_t code = 0;
int32_t curRow = 0;
int32_t lastRow = 0;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
TQ_NULL_GO_TO_END(assigned);
SArray* pCols = pSubmitTbData->aCol;
SColData* pCol = taosArrayGet(pCols, 0);
SArray* pCols = pSubmitTbData->aCol;
SColData* pCol = taosArrayGet(pCols, 0);
TQ_NULL_GO_TO_END(pCol);
int32_t numOfRows = pCol->nVal;
int32_t numOfCols = taosArrayGetSize(pCols);
int32_t numOfRows = pCol->nVal;
int32_t numOfCols = taosArrayGetSize(pCols);
for (int32_t i = 0; i < numOfRows; i++) {
bool buildNew = false;
for (int32_t j = 0; j < numOfCols; j++) {
pCol = taosArrayGet(pCols, j);
TQ_NULL_GO_TO_END(pCol);
SColVal colVal = {0};
SColVal colVal = {0};
tColDataGetValue(pCol, i, &colVal);
PROCESS_VAL
}
if (buildNew) {
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, curRow, &lastRow));
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows,
curRow, &lastRow));
}
SSDataBlock* pBlock = taosArrayGetLast(blocks);
@ -877,7 +878,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
TQ_NULL_GO_TO_END(pCol);
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
TQ_NULL_GO_TO_END(pColData);
SColVal colVal = {0};
SColVal colVal = {0};
tColDataGetValue(pCol, i, &colVal);
SET_DATA
}
@ -885,30 +886,30 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
curRow++;
}
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow;
pLastBlock->info.rows = curRow - lastRow;
END:
taosMemoryFree(assigned);
return code;
}
int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas){
int32_t code = 0;
STSchema* pTSchema = NULL;
int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) {
int32_t code = 0;
STSchema* pTSchema = NULL;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols);
TQ_NULL_GO_TO_END(assigned);
int32_t curRow = 0;
int32_t lastRow = 0;
SArray* pRows = pSubmitTbData->aRowP;
int32_t numOfRows = taosArrayGetSize(pRows);
pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version);
int32_t curRow = 0;
int32_t lastRow = 0;
SArray* pRows = pSubmitTbData->aRowP;
int32_t numOfRows = taosArrayGetSize(pRows);
pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version);
for (int32_t i = 0; i < numOfRows; i++) {
bool buildNew = false;
SRow* pRow = taosArrayGetP(pRows, i);
SRow* pRow = taosArrayGetP(pRows, i);
TQ_NULL_GO_TO_END(pRow);
for (int32_t j = 0; j < pTSchema->numOfCols; j++) {
@ -918,7 +919,8 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
}
if (buildNew) {
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows, curRow, &lastRow));
TQ_ERR_GO_TO_END(processBuildNew(pReader, pSubmitTbData, blocks, schemas, pSchemaWrapper, assigned, numOfRows,
curRow, &lastRow));
}
SSDataBlock* pBlock = taosArrayGetLast(blocks);
@ -932,7 +934,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
int32_t colActual = blockDataGetNumOfCols(pBlock);
while (targetIdx < colActual) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
SColVal colVal = {0};
SColVal colVal = {0};
TQ_ERR_GO_TO_END(tRowGet(pRow, pTSchema, sourceIdx, &colVal));
SET_DATA
}
@ -940,7 +942,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra
curRow++;
}
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
pLastBlock->info.rows = curRow - lastRow;
pLastBlock->info.rows = curRow - lastRow;
END:
taosMemoryFreeClear(pTSchema);
@ -950,10 +952,10 @@ END:
int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) {
tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
SSDataBlock* block = NULL;
SSDataBlock* block = NULL;
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if(pSubmitTbData == NULL){
if (pSubmitTbData == NULL) {
return terrno;
}
pReader->nextBlk++;
@ -1034,7 +1036,7 @@ bool tqCurrentBlockConsumed(const STqReader* pReader) { return pReader->msg.msgS
void tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
if (pKey && taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)) != 0){
if (pKey && taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)) != 0) {
tqError("failed to remove table uid:%" PRId64 " from hash", *pKey);
}
}
@ -1064,7 +1066,8 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
int32_t sz = taosArrayGetSize(tbUidList);
for (int32_t i = 0; i < sz; i++) {
int64_t* tbUid = (int64_t*)taosArrayGet(tbUidList, i);
if (tbUid && taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, tbUid, sizeof(int64_t), NULL, 0) != 0){
if (tbUid &&
taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, tbUid, sizeof(int64_t), NULL, 0) != 0) {
tqError("failed to add table uid:%" PRId64 " to hash", *tbUid);
continue;
}

32
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -92,7 +92,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
char formatBuf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(formatBuf, TSDB_OFFSET_LEN, pOffsetVal);
tqDebug("tmq poll: consumer:0x%" PRIx64
", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64,
", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. QID:0x%" PRIx64,
consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId);
return 0;
} else {
@ -117,7 +117,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1);
code = tqInitDataRsp(&dataRsp.common, *pOffsetVal);
if (code != 0){
if (code != 0) {
return code;
}
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId,
@ -145,7 +145,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
terrno = 0;
SMqDataRsp dataRsp = {0};
int code = tqInitDataRsp(&dataRsp.common, *pOffset);
int code = tqInitDataRsp(&dataRsp.common, *pOffset);
if (code != 0) {
goto end;
}
@ -176,7 +176,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
end : {
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, QID:0x%" PRIx64
" code:%d",
consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code);
tDeleteMqDataRsp(&dataRsp);
@ -206,10 +206,10 @@ static void tDeleteCommon(void* parm) {}
static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, STqOffsetVal* offset) {
int32_t vgId = TD_VID(pTq->pVnode);
STaosxRsp taosxRsp = {0};
SMqBatchMetaRsp btMetaRsp = {0};
int32_t code = 0;
int32_t vgId = TD_VID(pTq->pVnode);
STaosxRsp taosxRsp = {0};
SMqBatchMetaRsp btMetaRsp = {0};
int32_t code = 0;
TQ_ERR_GO_TO_END(tqInitTaosxRsp(&taosxRsp.common, *offset));
if (offset->type != TMQ_OFFSET__LOG) {
@ -323,9 +323,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
continue;
}
int32_t tLen = sizeof(SMqRspHead) + len;
void* tBuf = taosMemoryCalloc(1, tLen);
if (tBuf == NULL){
int32_t tLen = sizeof(SMqRspHead) + len;
void* tBuf = taosMemoryCalloc(1, tLen);
if (tBuf == NULL) {
code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
goto END;
}
@ -339,11 +339,11 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
continue;
}
if (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf) == NULL){
if (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf) == NULL) {
code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
goto END;
}
if (taosArrayPush(btMetaRsp.batchMetaLen, &tLen) == NULL){
if (taosArrayPush(btMetaRsp.batchMetaLen, &tLen) == NULL) {
code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
goto END;
}
@ -601,7 +601,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void*
SColumnInfoData* pUidCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX);
TSDB_CHECK_NULL(pUidCol, code, line, END, terrno)
int64_t* pUid = taosArrayGet(pRes->uidList, i);
int64_t* pUid = taosArrayGet(pRes->uidList, i);
code = colDataSetVal(pUidCol, i, (const char*)pUid, false);
TSDB_CHECK_CODE(code, line, END);
void* tmp = taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX);
@ -632,7 +632,7 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void*
}
END:
if (code != 0){
if (code != 0) {
tqError("failed to extract delete data block, line:%d code:%d", line, code);
}
tDecoderClear(pCoder);
@ -662,7 +662,7 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
for (int32_t i = 0; i < numOfTasks; ++i) {
SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i);
if (pId == NULL){
if (pId == NULL) {
continue;
}
if (pId->streamId != streamId) {

73
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -207,7 +207,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
updated = streamTaskUpdateEpsetInfo(pTask, req.pNodeList);
// send the checkpoint-source-rsp for source task to end the checkpoint trans in mnode
(void) streamTaskSendCheckpointsourceRsp(pTask);
(void)streamTaskSendCheckpointsourceRsp(pTask);
streamTaskResetStatus(pTask);
streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr);
@ -303,7 +303,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList);
return rsp.code; // always return true
return rsp.code; // always return true
}
int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -324,7 +324,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId);
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask && (code == 0)) {
SRpcMsg rsp = {.info = pMsg->info, .code = 0};
if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) {
@ -384,7 +384,7 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
pRsp->downstreamTaskId, pRsp->downstreamNodeId);
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
if (pTask && (code == 0)) {
code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
streamMetaReleaseTask(pMeta, pTask);
@ -482,8 +482,8 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
}
tDecoderClear(&decoder);
tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d",
rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);
tqDebug("tq task:0x%x (vgId:%d) recv check rsp(QID:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId,
rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);
if (!isLeader) {
tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId,
@ -681,7 +681,8 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId,
(int32_t)hTaskId.taskId);
}
}
@ -701,7 +702,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
}
streamMetaWUnLock(pMeta);
return 0; // always return success
return 0; // always return success
}
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) {
@ -793,23 +794,23 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
int32_t vgId = pMeta->vgId;
if (type == STREAM_EXEC_T_START_ONE_TASK) {
(void) streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
(void)streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
return 0;
} else if (type == STREAM_EXEC_T_START_ALL_TASKS) {
(void) streamMetaStartAllTasks(pMeta);
(void)streamMetaStartAllTasks(pMeta);
return 0;
} else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) {
(void) restartStreamTasks(pMeta, isLeader);
(void)restartStreamTasks(pMeta, isLeader);
return 0;
} else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) {
(void) streamMetaStopAllTasks(pMeta);
(void)streamMetaStopAllTasks(pMeta);
return 0;
} else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
return code;
} else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL && (code == 0)) {
char* pStatus = NULL;
@ -831,13 +832,13 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
}
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed
char* p = NULL;
if (streamTaskReadyToRun(pTask, &p)) {
tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId,
pTask->id.idStr, p, pTask->chkInfo.nextProcessVer);
(void) streamExecTask(pTask);
(void)streamExecTask(pTask);
} else {
int8_t status = streamTaskSetSchedStatusInactive(pTask);
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
@ -900,7 +901,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->taskId);
@ -924,9 +925,9 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
streamTaskSetStatusReady(pTask);
} else if (pState.state == TASK_STATUS__UNINIT) {
// tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
// ASSERT(pTask->status.downstreamReady == 0);
// tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
// tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
// ASSERT(pTask->status.downstreamReady == 0);
// tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
} else {
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
@ -942,7 +943,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
" from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
@ -958,7 +959,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_STREAM_TASK_IVLD_STATUS);
TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamMetaReleaseTask(pMeta, pTask);
return code;
}
@ -996,7 +997,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
pTask->id.idStr, recv, total);
}
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
TSDB_CODE_ACTION_IN_PROGRESS);
}
} else { // upstream not recv the checkpoint-source/trigger till now
ASSERT(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT);
@ -1005,7 +1006,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"upstream sending checkpoint-source/trigger",
pTask->id.idStr);
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
TSDB_CODE_ACTION_IN_PROGRESS);
}
streamMetaReleaseTask(pMeta, pTask);
@ -1016,7 +1017,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError(
"vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
@ -1038,7 +1039,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
pReq->taskId);
@ -1122,7 +1123,7 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
@ -1169,13 +1170,15 @@ int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { ret
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) {
return doProcessDummyRspMsg(pMeta, pMsg);
}
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped",
pRsp->downstreamNodeId, pRsp->downstreamTaskId);
@ -1211,8 +1214,9 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId);
tqError(
"vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId);
(void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
return code;
}
@ -1221,7 +1225,8 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
if (req.startTs < pTask->execInfo.created) {
tqWarn("s-task:%s vgId:%d create time:%" PRId64 " recv expired consensus checkpointId:%" PRId64
" from task createTs:%" PRId64 " < task createTs:%" PRId64 ", discard",
pTask->id.idStr, pMeta->vgId, pTask->execInfo.created, req.checkpointId, req.startTs, pTask->execInfo.created);
pTask->id.idStr, pMeta->vgId, pTask->execInfo.created, req.checkpointId, req.startTs,
pTask->execInfo.created);
streamMetaAddFailedTaskSelf(pTask, now);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
@ -1234,15 +1239,15 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId);
if (pTask->chkInfo.consensusTransId >= req.transId) {
tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard",
pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId);
tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", pTask->id.idStr, vgId,
pTask->chkInfo.consensusTransId, req.transId);
streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
}
if (pTask->chkInfo.checkpointId != req.checkpointId) {
tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64" transId:%d", pTask->id.idStr, vgId,
tqDebug("s-task:%s vgId:%d update the checkpoint from %" PRId64 " to %" PRId64 " transId:%d", pTask->id.idStr, vgId,
pTask->chkInfo.checkpointId, req.checkpointId, req.transId);
pTask->chkInfo.checkpointId = req.checkpointId;
tqSetRestoreVersionInfo(pTask);

411
source/libs/catalog/src/ctgAsync.c
View File

File diff suppressed because it is too large Load Diff

76
source/libs/catalog/src/ctgRemote.c
View File

@ -57,13 +57,13 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
}
for (int32_t i = 0; i < taskNum; ++i) {
int32_t* taskId = taosArrayGet(cbParam->taskId, i);
int32_t* taskId = taosArrayGet(cbParam->taskId, i);
if (NULL == taskId) {
ctgError("taosArrayGet %d taskId failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->taskId));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i);
int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i);
if (NULL == msgIdx) {
ctgError("taosArrayGet %d msgIdx failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->msgIdx));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
@ -80,7 +80,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
if (pRsp->msgIdx != *msgIdx) {
ctgError("rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx);
pRsp = &rsp;
pRsp->msgIdx = *msgIdx;
pRsp->reqType = -1;
@ -117,7 +117,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId,
pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs);
(void)(*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal
(void)(*gCtgAsyncFps[pTask->type].handleRspFp)(
&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal
}
CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs));
@ -420,12 +421,12 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
if (TDMT_VND_BATCH_META == cbParam->reqType || TDMT_MND_BATCH_META == cbParam->reqType) {
CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode));
} else {
int32_t* taskId = taosArrayGet(cbParam->taskId, 0);
int32_t* taskId = taosArrayGet(cbParam->taskId, 0);
if (NULL == taskId) {
ctgError("taosArrayGet %d taskId failed, total:%d", 0, (int32_t)taosArrayGetSize(cbParam->taskId));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId);
if (NULL == pTask) {
ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks));
@ -448,7 +449,7 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
ctgError("get task %d SCtgMsgCtx failed, taskType:%d", -1, pTask->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pMsgCtx->pBatchs = pBatchs;
#endif
@ -537,7 +538,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob,
CTG_ERR_JRET(code);
}
ctgDebug("ctg req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType));
ctgDebug("ctg req msg sent, QID:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType));
return TSDB_CODE_SUCCESS;
_return:
@ -561,9 +562,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq->msgIdx, pTask->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SHashObj* pBatchs = pMsgCtx->pBatchs;
SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId));
SHashObj* pBatchs = pMsgCtx->pBatchs;
SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId));
if (NULL == pBatch) {
newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg));
newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t));
@ -602,7 +603,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName));
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){
} else if (CTG_TASK_GET_TB_TSMA == pTask->type) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
@ -619,10 +620,11 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches));
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx,
(int32_t)taosArrayGetSize(pCtx->pFetches));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
if (NULL == pTbReq) {
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
@ -678,7 +680,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName));
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){
} else if (CTG_TASK_GET_TB_TSMA == pTask->type) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
@ -692,22 +694,23 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
pName = ctx->pName;
}
} else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
if (NULL == pTbReq) {
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (NULL == pName) {
ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx,
(int32_t)taosArrayGetSize(pCtx->pFetches));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
if (NULL == pTbReq) {
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (NULL == pName) {
ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
} else {
ctgError("invalid vnode msgType %d", msgType);
CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
@ -1632,9 +1635,9 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName*
}
int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* name, STableTSMAInfoRsp* out,
SCtgTaskReq* tReq, int32_t reqType) {
char* msg = NULL;
int32_t msgLen = 0;
SCtgTaskReq* tReq, int32_t reqType) {
char* msg = NULL;
int32_t msgLen = 0;
SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char tbFName[TSDB_TABLE_FNAME_LEN];
@ -1723,7 +1726,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c
#if CTG_BATCH_FETCH
CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen));
#else
char dbFName[TSDB_DB_FNAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
(void)tNameGetFullDbName(pTbName, dbFName);
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) {
@ -1734,7 +1737,8 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen));
CTG_RET(
ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen));
#endif
}

98
source/libs/stream/src/streamCheckStatus.c
View File

@ -36,7 +36,7 @@ static void setCheckDownstreamReqInfo(SStreamTaskCheckReq* pReq, int64_t reqId,
static void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, int32_t* numOfFault,
int32_t* numOfNotRsp, SArray* pTimeoutList, SArray* pNotReadyList, const char* id);
static int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId);
static void findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId, SDownstreamStatusInfo** pStatusInfo);
static void findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId, SDownstreamStatusInfo** pStatusInfo);
int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage,
int64_t* oldStage) {
@ -107,11 +107,12 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) {
streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pDispatch->taskId, pDispatch->nodeId, idstr);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " check single downstream task:0x%x(vgId:%d) ver:%" PRId64 "-%" PRId64
" window:%" PRId64 "-%" PRId64 " reqId:0x%" PRIx64,
" window:%" PRId64 "-%" PRId64 " QID:0x%" PRIx64,
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer,
pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId);
(void) streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet);
(void)streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId,
&pTask->outputInfo.fixedDispatcher.epSet);
} else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
streamTaskStartMonitorCheckRsp(pTask);
@ -132,9 +133,9 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) {
streamTaskAddReqInfo(&pTask->taskCheckInfo, req.reqId, pVgInfo->taskId, pVgInfo->vgId, idstr);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64
" check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64,
" check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, QID:0x%" PRIx64,
idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId);
(void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
(void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
}
} else { // for sink task, set it ready directly.
stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId);
@ -164,20 +165,20 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS
pRsp->status = TASK_DOWNSTREAM_NOT_LEADER;
} else {
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, taskId, &pTask);
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, taskId, &pTask);
if (pTask != NULL) {
pRsp->status =
streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage);
SStreamTaskState pState = streamTaskGetStatus(pTask);
stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64
stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(QID:0x%" PRIx64
") task:0x%x (vgId:%d), check_status:%d",
pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId,
pRsp->status);
streamMetaReleaseTask(pMeta, pTask);
} else {
pRsp->status = TASK_DOWNSTREAM_NOT_READY;
stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64
stDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(QID:0x%" PRIx64
") from task:0x%x (vgId:%d), rsp check_status %d",
pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status);
}
@ -261,7 +262,7 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tEncoderInit(&encoder, (uint8_t*)abuf, len);
(void) tEncodeStreamTaskCheckRsp(&encoder, pRsp);
(void)tEncodeStreamTaskCheckRsp(&encoder, pRsp);
tEncoderClear(&encoder);
SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo};
@ -300,7 +301,8 @@ void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) {
if (pInfo->checkRspTmr == NULL) {
pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer);
} else {
streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor");
streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId,
"check-status-monitor");
}
streamMutexUnlock(&pInfo->checkInfoLock);
@ -319,7 +321,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
pInfo->pList = NULL;
if (pInfo->checkRspTmr != NULL) {
(void) taosTmrStop(pInfo->checkRspTmr);
(void)taosTmrStop(pInfo->checkRspTmr);
pInfo->checkRspTmr = NULL;
}
@ -329,11 +331,11 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void processDownstreamReadyRsp(SStreamTask* pTask) {
EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
(void) streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL);
(void)streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL);
int64_t checkTs = pTask->execInfo.checkTs;
int64_t readyTs = pTask->execInfo.readyTs;
(void) streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true);
(void)streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true);
if (pTask->status.taskStatus == TASK_STATUS__HALT) {
if (!HAS_RELATED_FILLHISTORY_TASK(pTask) || (pTask->info.fillHistory != 0)) {
@ -355,7 +357,7 @@ void processDownstreamReadyRsp(SStreamTask* pTask) {
// todo: let's retry
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr);
(void) streamLaunchFillHistoryTask(pTask);
(void)streamLaunchFillHistoryTask(pTask);
}
}
@ -379,7 +381,7 @@ void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) {
if (!existed) {
SDownstreamTaskEpset t = {.nodeId = nodeId};
void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t);
void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t);
if (p == NULL) {
// todo let's retry
}
@ -430,7 +432,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
findCheckRspStatus(pInfo, taskId, &p);
if (p != NULL) {
if (reqId != p->reqId) {
stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64
stError("s-task:%s QID:0x%" PRIx64 " expected:0x%" PRIx64
" expired check-rsp recv from downstream task:0x%x, discarded",
id, reqId, p->reqId, taskId);
streamMutexUnlock(&pInfo->checkInfoLock);
@ -452,7 +454,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
}
streamMutexUnlock(&pInfo->checkInfoLock);
stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId,
stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, QID:%" PRIx64 " discarded", id, taskId,
reqId);
return TSDB_CODE_FAILED;
}
@ -539,10 +541,10 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher;
setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id,
stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) QID:0x%" PRIx64, id,
pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId);
(void) streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet);
(void)streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet);
} else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos;
int32_t numOfVgs = taosArrayGetSize(vgInfo);
@ -557,9 +559,9 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) {
setCheckDownstreamReqInfo(&req, p->reqId, pVgInfo->taskId, pVgInfo->vgId);
stDebug("s-task:%s (vgId:%d) stage:%" PRId64
" re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64,
" re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d QID:0x%" PRIx64,
id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId);
(void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
(void)streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
break;
}
}
@ -580,15 +582,15 @@ void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, i
stDebug("s-task:%s recv status:NEW_STAGE/NOT_LEADER from downstream, task:0x%x, quit from check downstream", id,
p->taskId);
(*numOfFault) += 1;
} else { // TASK_DOWNSTREAM_NOT_READY
if (p->rspTs == 0) { // not response yet
} else { // TASK_DOWNSTREAM_NOT_READY
if (p->rspTs == 0) { // not response yet
if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec.
(void) taosArrayPush(pTimeoutList, &p->taskId);
(void)taosArrayPush(pTimeoutList, &p->taskId);
} else { // el < CHECK_NOT_RSP_DURATION
(*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp
}
} else {
(void) taosArrayPush(pNotReadyList, &p->taskId);
(void)taosArrayPush(pNotReadyList, &p->taskId);
}
}
}
@ -613,14 +615,13 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) {
continue;
}
int32_t taskId = *px;
int32_t taskId = *px;
SDownstreamStatusInfo* p = NULL;
findCheckRspStatus(pInfo, taskId, &p);
if (p != NULL) {
if (p->status != -1 || p->rspTs != 0) {
stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%"PRId64, pTask->id.idStr, i, p->status,
p->rspTs);
stError("s-task:%s invalid rsp record entry, index:%d, status:%d, rspTs:%" PRId64, pTask->id.idStr, i,
p->status, p->rspTs);
continue;
}
@ -692,24 +693,24 @@ int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64
// this function is executed in timer thread
void rspMonitorFn(void* param, void* tmrId) {
SStreamTask* pTask = param;
SStreamMeta* pMeta = pTask->pMeta;
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
int32_t vgId = pTask->pMeta->vgId;
int64_t now = taosGetTimestampMs();
int64_t timeoutDuration = now - pInfo->timeoutStartTs;
const char* id = pTask->id.idStr;
int32_t numOfReady = 0;
int32_t numOfFault = 0;
int32_t numOfNotRsp = 0;
int32_t numOfNotReady = 0;
int32_t numOfTimeout = 0;
int32_t total = taosArrayGetSize(pInfo->pList);
SStreamTask* pTask = param;
SStreamMeta* pMeta = pTask->pMeta;
STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
int32_t vgId = pTask->pMeta->vgId;
int64_t now = taosGetTimestampMs();
int64_t timeoutDuration = now - pInfo->timeoutStartTs;
const char* id = pTask->id.idStr;
int32_t numOfReady = 0;
int32_t numOfFault = 0;
int32_t numOfNotRsp = 0;
int32_t numOfNotReady = 0;
int32_t numOfTimeout = 0;
int32_t total = taosArrayGetSize(pInfo->pList);
stDebug("s-task:%s start to do check-downstream-rsp check in tmr", id);
streamMutexLock(&pTask->lock);
SStreamTaskState state = streamTaskGetStatus(pTask);
SStreamTaskState state = streamTaskGetStatus(pTask);
streamMutexUnlock(&pTask->lock);
if (state.state == TASK_STATUS__STOP) {
@ -721,7 +722,7 @@ void rspMonitorFn(void* param, void* tmrId) {
// not record the failed of the current task if try to close current vnode
// otherwise, the put of message operation may incur invalid read of message queue.
if (!pMeta->closeFlag) {
(void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
(void)addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
}
streamMetaReleaseTask(pMeta, pTask);
@ -740,8 +741,8 @@ void rspMonitorFn(void* param, void* tmrId) {
streamMutexLock(&pInfo->checkInfoLock);
if (pInfo->notReadyTasks == 0) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, state.name,
vgId, ref);
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, state.name, vgId,
ref);
streamTaskCompleteCheckRsp(pInfo, false, id);
streamMutexUnlock(&pInfo->checkInfoLock);
@ -803,7 +804,7 @@ void rspMonitorFn(void* param, void* tmrId) {
streamTaskCompleteCheckRsp(pInfo, false, id);
streamMutexUnlock(&pInfo->checkInfoLock);
(void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
(void)addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId);
streamMetaReleaseTask(pMeta, pTask);
taosArrayDestroy(pNotReadyList);
@ -819,7 +820,8 @@ void rspMonitorFn(void* param, void* tmrId) {
handleTimeoutDownstreamTasks(pTask, pTimeoutList);
}
streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor");
streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId,
"check-status-monitor");
streamMutexUnlock(&pInfo->checkInfoLock);
stDebug(

51
source/libs/stream/src/streamDispatch.c
View File

@ -118,7 +118,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder;
tEncoderInit(&encoder, abuf, len);
(void) tEncodeStreamRetrieveReq(&encoder, req);
(void)tEncodeStreamRetrieveReq(&encoder, req);
tEncoderClear(&encoder);
SRpcMsg rpcMsg = {0};
@ -130,7 +130,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
return code;
}
stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr,
stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), QID:0x%" PRIx64, pTask->id.idStr,
pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId);
}
@ -254,7 +254,7 @@ static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStre
int32_t type = pTask->outputInfo.type;
int32_t num = streamTaskGetNumOfDownstream(pTask);
if(type != TASK_OUTPUT__SHUFFLE_DISPATCH && type != TASK_OUTPUT__FIXED_DISPATCH) {
if (type != TASK_OUTPUT__SHUFFLE_DISPATCH && type != TASK_OUTPUT__FIXED_DISPATCH) {
terrno = TSDB_CODE_INVALID_PARA;
stError("s-task:%s invalid dispatch type:%d not dispatch data", pTask->id.idStr, type);
return NULL;
@ -283,7 +283,7 @@ static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStre
return NULL;
}
}
} else { // shuffle dispatch
} else { // shuffle dispatch
int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId;
@ -470,7 +470,7 @@ static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t
streamMutexLock(&pMsgInfo->lock);
}
(void) taosArrayPush(pMsgInfo->pSendInfo, &entry);
(void)taosArrayPush(pMsgInfo->pSendInfo, &entry);
if (lock) {
streamMutexUnlock(&pMsgInfo->lock);
@ -591,7 +591,7 @@ static void doMonitorDispatchData(void* param, void* tmrId) {
SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet;
int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId;
int32_t s = taosArrayGetSize(pTask->msgInfo.pSendInfo);
int32_t s = taosArrayGetSize(pTask->msgInfo.pSendInfo);
SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, 0);
if (pEntry != NULL) {
setResendInfo(pEntry, now);
@ -617,7 +617,8 @@ static void doMonitorDispatchData(void* param, void* tmrId) {
void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) {
int32_t vgId = pTask->pMeta->vgId;
if (pTask->msgInfo.pRetryTmr != NULL) {
streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, "dispatch-monitor-tmr");
streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId,
"dispatch-monitor-tmr");
} else {
pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer);
}
@ -651,7 +652,8 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
}
}
} else {
(void) buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName);
(void)buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId,
pDataBlock->info.parTbName);
}
snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db,
@ -666,7 +668,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
// failed to put into name buffer, no need to do anything
if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) {
(void) tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName));
(void)tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName));
}
}
@ -848,7 +850,8 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
}
if (++pTmrInfo->activeCounter < 50) {
streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor");
streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
"chkpt-ready-monitor");
return;
}
@ -907,7 +910,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
continue;
}
(void) taosArrayPush(pNotRspList, &pInfo->upstreamTaskId);
(void)taosArrayPush(pNotRspList, &pInfo->upstreamTaskId);
stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId,
pTask->info.taskLevel, pInfo->upstreamTaskId);
}
@ -931,8 +934,8 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
if (*pTaskId == pReadyInfo->upstreamTaskId) { // send msg again
SRpcMsg msg = {0};
int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId,
checkpointId, &msg);
int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId,
pReadyInfo->childId, checkpointId, &msg);
if (code == TSDB_CODE_SUCCESS) {
code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg);
if (code == TSDB_CODE_SUCCESS) {
@ -948,7 +951,8 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
}
}
streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor");
streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
"chkpt-ready-monitor");
streamMutexUnlock(&pActiveInfo->lock);
} else {
int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask);
@ -1015,7 +1019,8 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
if (pTmrInfo->tmrHandle == NULL) {
pTmrInfo->tmrHandle = taosTmrStart(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer);
} else {
streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "chkpt-ready-monitor");
streamTmrReset(checkpointReadyMsgSendMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
"chkpt-ready-monitor");
}
// mark the timer monitor checkpointId
@ -1087,8 +1092,8 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch
payloadLen += sizeof(SRetrieveTableRsp);
(void) taosArrayPush(pReq->dataLen, &payloadLen);
(void) taosArrayPush(pReq->data, &buf);
(void)taosArrayPush(pReq->dataLen, &payloadLen);
(void)taosArrayPush(pReq->data, &buf);
pReq->totalLen += dataStrLen;
return 0;
@ -1169,7 +1174,7 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
tEncoderInit(&encoder, (uint8_t*)abuf, len);
(void) tEncodeStreamCheckpointSourceRsp(&encoder, &rsp);
(void)tEncodeStreamCheckpointSourceRsp(&encoder, &rsp);
tEncoderClear(&encoder);
initRpcMsg(pMsg, 0, pBuf, sizeof(SMsgHead) + len);
@ -1185,7 +1190,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
.recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId};
// todo retry until it success
(void) streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS);
(void)streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS);
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
streamMutexLock(&pActiveInfo->lock);
@ -1208,7 +1213,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId);
}
} else {
(void) taosArrayPush(pActiveInfo->pReadyMsgList, &info);
(void)taosArrayPush(pActiveInfo->pReadyMsgList, &info);
stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1);
}
@ -1217,7 +1222,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
}
void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId,
int32_t childId, SEpSet* pEpset, int64_t checkpointId) {
int32_t childId, SEpSet* pEpset, int64_t checkpointId) {
pReadyInfo->upstreamTaskId = upstreamTaskId;
pReadyInfo->upstreamNodeEpset = *pEpset;
pReadyInfo->upstreamNodeId = upstreamNodeId;
@ -1246,7 +1251,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
streamMutexLock(&pActiveInfo->lock);
(void) taosArrayPush(pActiveInfo->pReadyMsgList, &info);
(void)taosArrayPush(pActiveInfo->pReadyMsgList, &info);
int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList);
int32_t total = streamTaskGetNumOfUpstream(pTask);
@ -1282,7 +1287,7 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId
stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData);
int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs;
bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER);
bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER);
clearBufferedDispatchMsg(pTask);

81
source/libs/stream/src/streamExec.c
View File

@ -18,10 +18,10 @@
// maximum allowed processed block batches. One block may include several submit blocks
#define MAX_STREAM_EXEC_BATCH_NUM 32
#define STREAM_RESULT_DUMP_THRESHOLD 300
#define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1) // 1MiB result data
#define STREAM_SCAN_HISTORY_TIMESLICE 1000 // 1000 ms
#define MIN_INVOKE_INTERVAL 50 // 50ms
#define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec
#define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1) // 1MiB result data
#define STREAM_SCAN_HISTORY_TIMESLICE 1000 // 1000 ms
#define MIN_INVOKE_INTERVAL 50 // 50ms
#define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec
static int32_t streamTransferStateDoPrepare(SStreamTask* pTask);
static void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks);
@ -54,7 +54,7 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl
// not handle error, if dispatch failed, try next time.
// checkpoint trigger will be checked
(void) streamDispatchStreamBlock(pTask);
(void)streamDispatchStreamBlock(pTask);
}
return code;
@ -78,7 +78,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray*
}
stDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks,
SIZE_IN_MiB(size));
SIZE_IN_MiB(size));
code = doOutputResultBlockImpl(pTask, pStreamBlocks);
if (code != TSDB_CODE_SUCCESS) { // back pressure and record position
@ -99,7 +99,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
*totalSize = 0;
int32_t size = 0;
int32_t numOfBlocks= 0;
int32_t numOfBlocks = 0;
SArray* pRes = NULL;
while (1) {
@ -129,7 +129,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem;
ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1);
(void) assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0));
(void)assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0));
block.info.type = STREAM_PULL_OVER;
block.info.childId = pTask->info.selfChildId;
@ -140,8 +140,8 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
stError("s-task:%s failed to add retrieve block", pTask->id.idStr);
}
stDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64 " dump results", pTask->id.idStr,
pTask->info.selfChildId, pRetrieveBlock->reqId);
stDebug("s-task:%s(child %d) retrieve process completed, QID:0x%" PRIx64 " dump results", pTask->id.idStr,
pTask->info.selfChildId, pRetrieveBlock->reqId);
}
break;
@ -174,7 +174,7 @@ void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* to
}
stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr,
pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size));
pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size));
// current output should be dispatched to down stream nodes
if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) {
@ -251,7 +251,7 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t*
}
SSDataBlock block = {0};
(void) assignOneDataBlock(&block, output);
(void)assignOneDataBlock(&block, output);
block.info.childId = pTask->info.selfChildId;
void* p = taosArrayPush(pRes, &block);
@ -259,7 +259,8 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t*
stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr);
}
(*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
(*pSize) +=
blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block);
numOfBlocks += 1;
if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || (*pSize) >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) {
@ -283,7 +284,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
const char* id = pTask->id.idStr;
if (!pTask->hTaskInfo.operatorOpen) {
(void) qSetStreamOpOpen(exec);
(void)qSetStreamOpOpen(exec);
pTask->hTaskInfo.operatorOpen = true;
}
@ -315,13 +316,13 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
int32_t size = 0;
streamScanHistoryDataImpl(pTask, pRes, &size, &finished);
if(streamTaskShouldStop(pTask)) {
if (streamTaskShouldStop(pTask)) {
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0);
}
// dispatch the generated results, todo fix error
(void) handleScanhistoryResultBlocks(pTask, pRes, size);
(void)handleScanhistoryResultBlocks(pTask, pRes, size);
if (finished) {
return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0);
@ -346,11 +347,11 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
stError(
"s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related "
"fill-history task",
id, (int32_t) pTask->streamTaskId.taskId);
id, (int32_t)pTask->streamTaskId.taskId);
// 1. free it and remove fill-history task from disk meta-store
// todo: this function should never be failed.
(void) streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
(void)streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0);
// 2. save to disk
streamMetaWLock(pMeta);
@ -367,7 +368,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
id, streamTaskGetStatus(pTask).name, el, pStreamTask->id.idStr);
}
ETaskStatus status = streamTaskGetStatus(pStreamTask).state;
ETaskStatus status = streamTaskGetStatus(pStreamTask).state;
STimeWindow* pTimeWindow = &pStreamTask->dataRange.window;
// It must be halted for a source stream task, since when the related scan-history-data task start scan the history
@ -408,14 +409,14 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN,
pTimeWindow->ekey, p, pStreamTask->status.schedStatus);
(void) streamTaskResetTimewindowFilter(pStreamTask);
(void)streamTaskResetTimewindowFilter(pStreamTask);
} else {
stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr);
}
// NOTE: transfer the ownership of executor state before handle the checkpoint block during stream exec
// 2. send msg to mnode to launch a checkpoint to keep the state for current stream
(void) streamTaskSendCheckpointReq(pStreamTask);
(void)streamTaskSendCheckpointReq(pStreamTask);
// 3. assign the status to the value that will be kept in disk
pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state;
@ -429,12 +430,12 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
static int32_t haltCallback(SStreamTask* pTask, void* param) {
streamTaskOpenAllUpstreamInput(pTask);
(void) streamTaskSendCheckpointReq(pTask);
(void)streamTaskSendCheckpointReq(pTask);
return TSDB_CODE_SUCCESS;
}
int32_t streamTransferStatePrepare(SStreamTask* pTask) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t code = TSDB_CODE_SUCCESS;
SStreamMeta* pMeta = pTask->pMeta;
ASSERT(pTask->status.appendTranstateBlock == 1);
@ -466,7 +467,7 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) {
// set input
static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) {
void* pExecutor = pTask->exec.pExecutor;
void* pExecutor = pTask->exec.pExecutor;
int32_t code = 0;
const SStreamQueueItem* pItem = pInput;
@ -479,7 +480,7 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int
const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput;
code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT);
stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit,
pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver);
pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver);
ASSERT((*pVer) <= pSubmit->submit.ver);
(*pVer) = pSubmit->submit.ver;
@ -497,7 +498,7 @@ static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int
SArray* pBlockList = pMerged->submits;
int32_t numOfBlocks = taosArrayGetSize(pBlockList);
stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks,
pMerged->ver);
pMerged->ver);
code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT);
ASSERT((*pVer) <= pMerged->ver);
(*pVer) = pMerged->ver;
@ -549,7 +550,7 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock)
pBlock->srcVgId = pTask->pMeta->vgId;
code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock);
if (code == 0) {
(void) streamDispatchStreamBlock(pTask);
(void)streamDispatchStreamBlock(pTask);
} else { // todo put into queue failed, retry
streamFreeQitem((SStreamQueueItem*)pBlock);
}
@ -568,7 +569,7 @@ void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock)
}
}
//static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; }
// static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; }
static void setLastExecTs(SStreamTask* pTask, int64_t ts) { pTask->status.lastExecTs = ts; }
static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, int32_t num) {
@ -581,7 +582,7 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i
stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type));
int32_t code = doSetStreamInputBlock(pTask, pBlock, &ver, id);
if(code) {
if (code) {
stError("s-task:%s failed to set input block, not exec for these blocks", id);
return;
}
@ -609,7 +610,7 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i
if (ver != pInfo->processedVer) {
stDebug("s-task:%s update processedVer(unsaved) from %" PRId64 " to %" PRId64 " nextProcessVer:%" PRId64
" ckpt:%" PRId64,
" ckpt:%" PRId64,
id, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer);
pInfo->processedVer = ver;
}
@ -625,10 +626,10 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB
STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = NULL;
int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (code == TSDB_CODE_SUCCESS) { // ignore the error code.
(void) streamTaskReleaseState(pHTask);
(void) streamTaskReloadState(pTask);
int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (code == TSDB_CODE_SUCCESS) { // ignore the error code.
(void)streamTaskReleaseState(pHTask);
(void)streamTaskReloadState(pTask);
stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr,
streamTaskGetStatus(pHTask).name);
// todo execute qExecTask to fetch the reload-generated result, if this is stream is for session window query.
@ -707,7 +708,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
// dispatch checkpoint msg to all downstream tasks
int32_t type = pInput->type;
if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) {
(void) streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput);
(void)streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput);
continue;
}
@ -744,14 +745,14 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
if (type != STREAM_INPUT__CHECKPOINT) {
doStreamTaskExecImpl(pTask, pInput, numOfBlocks);
streamFreeQitem(pInput);
} else { // todo other thread may change the status
// do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed.
} else { // todo other thread may change the status
// do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed.
streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state == TASK_STATUS__CK) {
stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name);
(void) streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue
} else { // todo refactor
(void)streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue
} else { // todo refactor
int32_t code = 0;
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
code = streamTaskSendCheckpointSourceRsp(pTask);
@ -804,7 +805,7 @@ void streamResumeTask(SStreamTask* pTask) {
const char* id = pTask->id.idStr;
while (1) {
(void) doStreamExecTask(pTask);
(void)doStreamExecTask(pTask);
// check if continue
streamMutexLock(&pTask->lock);

2
source/libs/stream/src/streamTask.c
View File

@ -1092,7 +1092,7 @@ static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq*
}
// enqueue
stDebug("s-task:%s (vgId:%d level:%d) recv retrieve req from task:0x%x(vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr,
stDebug("s-task:%s (vgId:%d level:%d) recv retrieve req from task:0x%x(vgId:%d), QID:0x%" PRIx64, pTask->id.idStr,
pTask->pMeta->vgId, pTask->info.taskLevel, pReq->srcTaskId, pReq->srcNodeId, pReq->reqId);
pData->type = STREAM_INPUT__DATA_RETRIEVE;

12
source/libs/transport/inc/transLog.h
View File

@ -32,12 +32,12 @@ extern "C" {
#define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }}
#define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } }
#define tGTrace(param, ...) do { if (rpcDebugFlag & DEBUG_TRACE){char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", gtid:%s", __VA_ARGS__, buf);}} while(0)
#define tGFatal(param, ...) do {if (rpcDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tFatal(param ", gtid:%s", __VA_ARGS__, buf); }} while (0)
#define tGError(param, ...) do { if (rpcDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tError(param ", gtid:%s", __VA_ARGS__, buf);} } while(0)
#define tGWarn(param, ...) do { if (rpcDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tWarn(param ", gtid:%s", __VA_ARGS__, buf); }} while(0)
#define tGInfo(param, ...) do { if (rpcDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tInfo(param ", gtid:%s", __VA_ARGS__, buf); }} while(0)
#define tGDebug(param,...) do {if (rpcDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tDebug(param ", gtid:%s", __VA_ARGS__, buf); }} while(0)
#define tGTrace(param, ...) do { if (rpcDebugFlag & DEBUG_TRACE){char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", QID:%s", __VA_ARGS__, buf);}} while(0)
#define tGFatal(param, ...) do {if (rpcDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tFatal(param ", QID:%s", __VA_ARGS__, buf); }} while (0)
#define tGError(param, ...) do { if (rpcDebugFlag & DEBUG_ERROR){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tError(param ", QID:%s", __VA_ARGS__, buf);} } while(0)
#define tGWarn(param, ...) do { if (rpcDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tWarn(param ", QID:%s", __VA_ARGS__, buf); }} while(0)
#define tGInfo(param, ...) do { if (rpcDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tInfo(param ", QID:%s", __VA_ARGS__, buf); }} while(0)
#define tGDebug(param,...) do {if (rpcDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tDebug(param ", QID:%s", __VA_ARGS__, buf); }} while(0)
// clang-format on