diff --git a/example/src/tmq.c b/example/src/tmq.c index 4847e243a1..88fce7f4be 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -141,7 +141,7 @@ int32_t create_topic() { return 0; } -void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { +void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) { printf("commit %d\n", resp); } @@ -163,7 +163,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.db", "abc1"); tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); - tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); return tmq; } @@ -189,7 +189,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { cnt++; /*printf("get data\n");*/ /*msg_process(tmqmessage);*/ - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); /*} else {*/ /*break;*/ } @@ -219,7 +219,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); if (tmqmessage) { msg_process(tmqmessage); - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } @@ -249,7 +249,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) { batchCnt++; /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/ /*msg_process(tmqmessage);*/ - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); } else { break; } diff --git a/include/client/taos.h b/include/client/taos.h index 8b3fd14118..526edae8af 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -217,17 +217,17 @@ typedef struct tmq_conf_t tmq_conf_t; typedef struct tmq_list_t tmq_list_t; // typedef struct tmq_message_t tmq_message_t; -typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); +typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *)); DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT void tmq_list_destroy(tmq_list_t *); -#if 1 +#if 0 DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); #endif -DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen); +DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); @@ -271,14 +271,19 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); DLL_EXPORT char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); +// TODO +#if 0 +DLL_EXPORT char *tmq_get_block_table_name(TAOS_RES *res); +#endif + #if 0 DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); DLL_EXPORT int64_t tmq_get_request_offset(tmq_message_t *message); DLL_EXPORT int64_t tmq_get_response_offset(tmq_message_t *message); DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic); DLL_EXPORT int32_t tmq_field_count(tmq_t *tmq, const char *topic); -#endif DLL_EXPORT void tmq_message_destroy(TAOS_RES *res); +#endif /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */ #if 0 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a06b102458..809390a9d2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -331,6 +331,8 @@ typedef struct { int32_t pid; char app[TSDB_APP_NAME_LEN]; char db[TSDB_DB_NAME_LEN]; + char user[TSDB_USER_LEN]; + char passwd[TSDB_PASSWORD_LEN]; int64_t startTime; } SConnectReq; @@ -482,7 +484,7 @@ typedef struct { char intervalUnit; char slidingUnit; char - offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. + offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. int8_t precision; int64_t interval; int64_t sliding; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index eba70b8098..851d4d63c5 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -30,6 +30,7 @@ typedef struct SPlanContext { SNode* pAstRoot; bool topicQuery; bool streamQuery; + bool rSmaQuery; bool showRewrite; int8_t triggerType; int64_t watermark; diff --git a/packaging/install.sh b/packaging/install.sh index 9ce7fc326a..740d356f80 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -30,6 +30,15 @@ bin_dir="/usr/local/taos/bin" service_config_dir="/etc/systemd/system" +#taos-tools para +demoName="taosdemo" +benchmarkName="taosBenchmark" +dumpName="taosdump" +emailName="taosdata.com" +taosName="taos" +toolsName="taostools" + + # Color setting RED='\033[0;31m' GREEN='\033[1;32m' @@ -230,8 +239,20 @@ function install_header() { # temp install taosBenchmark function install_taosTools() { - cd ${script_dir}/taos-tools/ - tar xvf taosTools-1.4.1-Linux-x64.tar.gz && cd taosTools-1.4.1/ && ./install-taostools.sh + ${csudo} rm -f ${bin_link_dir}/${benchmarkName} || : + ${csudo} rm -f ${bin_link_dir}/${dumpName} || : + ${csudo} rm -f ${bin_link_dir}/rm${toolsName} || : + + ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${dumpName} ${install_main_dir}/bin/${dumpName} + ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${benchmarkName} + ${csudo} ln -sf ${install_main_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${demoName} + #Make link + [[ -x ${install_main_dir}/bin/${benchmarkName} ]] && \ + ${csudo} ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : + [[ -x ${install_main_dir}/bin/${demoName} ]] && \ + ${csudo} ln -s ${install_main_dir}/bin/${demoName} ${bin_link_dir}/${demoName} || : + [[ -x ${install_main_dir}/bin/${dumpName} ]] && \ + ${csudo} ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : } function add_newHostname_to_hosts() { diff --git a/packaging/release.sh b/packaging/release.sh index adf1195e56..a56d991bf8 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -39,7 +39,7 @@ cd ${compile_dir} echo "compile_dir: ${compile_dir}" -cmake .. +cmake .. -DBUILD_TOOLS=true make -j32 release_dir="${top_dir}/release" @@ -55,7 +55,6 @@ mkdir -p ${install_dir} mkdir -p ${install_dir}/bin mkdir -p ${install_dir}/lib mkdir -p ${install_dir}/inc -mkdir -p ${install_dir}/taos-tools install_files="${script_dir}/install.sh" chmod a+x ${script_dir}/install.sh || : @@ -64,13 +63,14 @@ cp ${install_files} ${install_dir} header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h" cp ${header_files} ${install_dir}/inc -bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos ${compile_dir}/build/bin/create_table ${compile_dir}/build/bin/tmq_sim ${script_dir}/remove.sh" -cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +bin_files="${compile_dir}/source/dnode/mgmt/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table ${compile_dir}/tests/test/c/tmq_sim ${script_dir}/remove.sh ${compile_dir}/build/bin/taosBenchmark ${compile_dir}/build/bin/taosdump" +cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : + +cp -rf ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/ +cp -rf ${compile_dir}/source/libs/tdb/libtdb.so ${install_dir}/lib/ +cp -rf ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries" +cp -rf ${compile_dir}/build/lib/pkgconfig ${install_dir}/lib/ > /dev/null || echo -e "failed to copy pkgconfig directory" -cp ${compile_dir}/build/lib/libtaos.so ${install_dir}/lib/ -cp ${compile_dir}/build/lib/libtdb.so ${install_dir}/lib/ -taostoolfile="${top_dir}/tools/taosTools-1.4.1-Linux-x64.tar.gz" -cp ${taostoolfile} ${install_dir}/taos-tools #cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/ #cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/ diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 35932cc13e..00a58e16a7 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -306,7 +306,6 @@ void hbMgrInitMqHbRspHandle(); SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery); - #ifdef __cplusplus } #endif diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0011c8baf3..10edb38bf1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -146,7 +146,8 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* (*pRequest)->sqlstr[sqlLen] = 0; (*pRequest)->sqlLen = sqlLen; - if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) { + if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, + sizeof((*pRequest)->self))) { destroyRequest(*pRequest); *pRequest = NULL; tscError("put request to request hash failed"); @@ -263,7 +264,8 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) { - if (precision != TSDB_TIME_PRECISION_MILLI && precision != TSDB_TIME_PRECISION_MICRO && precision != TSDB_TIME_PRECISION_NANO) { + if (precision != TSDB_TIME_PRECISION_MILLI && precision != TSDB_TIME_PRECISION_MICRO && + precision != TSDB_TIME_PRECISION_NANO) { return; } @@ -275,7 +277,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, &res); + pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); @@ -300,7 +302,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) { - SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { @@ -328,7 +330,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code if (!keepQuery) { qDestroyQuery(pQuery); } - + if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } @@ -336,7 +338,6 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code return pRequest; } - SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; @@ -522,6 +523,8 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType) { connectReq.pid = htonl(appInfo.pid); connectReq.startTime = htobe64(appInfo.startTime); tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); + tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user)); + tstrncpy(connectReq.passwd, pObj->pass, sizeof(connectReq.passwd)); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); void* pReq = taosMemoryMalloc(contLen); @@ -726,7 +729,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p)); ASSERT(len <= bytes); ASSERT((p + len) < (pResultInfo->convertBuf[i] + colLength[i])); - + varDataSetLen(p, len); pCol->offset[j] = (p - pResultInfo->convertBuf[i]); p += (len + VARSTR_HEADER_SIZE); @@ -744,51 +747,55 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int } pResultInfo->convertBuf[i] = p; - int32_t len = 0; + int32_t len = 0; SResultColumn* pCol = &pResultInfo->pCol[i]; for (int32_t j = 0; j < numOfRows; ++j) { if (pCol->offset[j] != -1) { char* pStart = pCol->offset[j] + pCol->pData; int32_t jsonInnerType = *pStart; - char *jsonInnerData = pStart + CHAR_BYTES; - char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; - if(jsonInnerType == TSDB_DATA_TYPE_NULL){ + char* jsonInnerData = pStart + CHAR_BYTES; + char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; + if (jsonInnerType == TSDB_DATA_TYPE_NULL) { sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(dst, strlen(varDataVal(dst))); - }else if(jsonInnerType == TSDB_DATA_TYPE_JSON){ - int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst)); + } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { + int32_t length = + taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst)); if (length <= 0) { - tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData)); + tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + varDataVal(jsonInnerData)); length = 0; } varDataSetLen(dst, length); - }else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" + } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" *(char*)varDataVal(dst) = '\"'; - int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst) + CHAR_BYTES); + int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), + varDataVal(dst) + CHAR_BYTES); if (length <= 0) { - tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData)); + tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + varDataVal(jsonInnerData)); length = 0; } - varDataSetLen(dst, length + CHAR_BYTES*2); + varDataSetLen(dst, length + CHAR_BYTES * 2); *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"'; - }else if(jsonInnerType == TSDB_DATA_TYPE_DOUBLE){ + } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { double jsonVd = *(double*)(jsonInnerData); sprintf(varDataVal(dst), "%.9lf", jsonVd); varDataSetLen(dst, strlen(varDataVal(dst))); - }else if(jsonInnerType == TSDB_DATA_TYPE_BIGINT){ + } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) { int64_t jsonVd = *(int64_t*)(jsonInnerData); sprintf(varDataVal(dst), "%" PRId64, jsonVd); varDataSetLen(dst, strlen(varDataVal(dst))); - }else if(jsonInnerType == TSDB_DATA_TYPE_BOOL){ - sprintf(varDataVal(dst), "%s", (*((char *)jsonInnerData) == 1) ? "true" : "false"); + } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { + sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); varDataSetLen(dst, strlen(varDataVal(dst))); - }else { + } else { ASSERT(0); } - if(len + varDataTLen(dst) > colLength[i]){ + if (len + varDataTLen(dst) > colLength[i]) { p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst)); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index d404edac91..14c7e9533d 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -24,6 +24,7 @@ #include "tqueue.h" #include "tref.h" +#if 0 struct tmq_message_t { SMqPollRsp msg; char* topic; @@ -31,6 +32,7 @@ struct tmq_message_t { int32_t vgId; int32_t resIter; }; +#endif typedef struct { int8_t tmqRspType; @@ -52,9 +54,7 @@ struct tmq_topic_vgroup_t { }; struct tmq_topic_vgroup_list_t { - int32_t cnt; - int32_t size; - tmq_topic_vgroup_t* elems; + SArray container; // SArray }; struct tmq_conf_t { @@ -63,6 +63,7 @@ struct tmq_conf_t { int8_t autoCommit; int8_t resetOffset; uint16_t port; + uint16_t autoCommitInterval; char* ip; char* user; char* pass; @@ -202,6 +203,11 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } + if (strcmp(key, "auto.commit.interval.ms") == 0) { + conf->autoCommitInterval = atoi(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "auto.offset.reset") == 0) { if (strcmp(value, "none") == 0) { conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; @@ -300,7 +306,7 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; if (pParam->tmq->commit_cb) { - pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL, NULL); + pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL); } if (!pParam->async) tsem_post(&pParam->rspSem); return 0; @@ -322,6 +328,7 @@ tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) { return tmq_subscribe(tmq, lst); } +#if 0 tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(sizeof(tmq_t), 1); if (pTmq == NULL) { @@ -357,8 +364,9 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return pTmq; } +#endif -tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { +tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { return NULL; @@ -369,6 +377,7 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { ASSERT(user); ASSERT(pass); ASSERT(conf->db); + ASSERT(conf->groupId[0]); pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ); if (pTmq->pTscObj == NULL) return NULL; @@ -429,8 +438,8 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in req.num = pArray->size; req.offsets = pArray->pData; } else { - req.num = offsets->cnt; - req.offsets = (SMqOffset*)offsets->elems; + req.num = taosArrayGetSize(&offsets->container); + req.offsets = (SMqOffset*)offsets->container.pData; } SCoder encoder; @@ -1538,16 +1547,6 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v } #endif -#if 0 -void tmq_message_destroy(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return; - SMqPollRsp* pRsp = &tmq_message->msg; - tDeleteSMqConsumeRsp(pRsp); - /*taosMemoryFree(tmq_message);*/ - taosFreeQitem(tmq_message); -} -#endif - tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; } const char* tmq_err2str(tmq_resp_err_t err) { diff --git a/source/client/test/tmqTest.cpp b/source/client/test/tmqTest.cpp index d45050d7ff..00ed16944c 100644 --- a/source/client/test/tmqTest.cpp +++ b/source/client/test/tmqTest.cpp @@ -108,7 +108,7 @@ TEST(testCase, tmq_subscribe_ctb_Test) { while (1) { tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); - tmq_message_destroy(msg); + taos_free_result(msg); //printf("get msg\n"); //if (msg == NULL) break; } @@ -141,7 +141,7 @@ TEST(testCase, tmq_subscribe_stb_Test) { tmq_commit(tmq, NULL, 0); } //tmq_commit(tmq, NULL, 0); - tmq_message_destroy(msg); + taos_free_result(msg); //printf("get msg\n"); } } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8b48d7914a..3622390f45 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2756,6 +2756,8 @@ int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) { if (tEncodeI32(&encoder, pReq->pid) < 0) return -1; if (tEncodeCStr(&encoder, pReq->app) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->passwd) < 0) return -1; if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1; tEndEncode(&encoder); @@ -2773,6 +2775,8 @@ int32_t tDeserializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) { if (tDecodeI32(&decoder, &pReq->pid) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->app) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->passwd) < 0) return -1; if (tDecodeI64(&decoder, &pReq->startTime) < 0) return -1; tEndDecode(&decoder); @@ -3032,7 +3036,6 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq return 0; } - int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3052,7 +3055,7 @@ int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq SReplica *pReplica = &pReq->replicas[i]; if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; } - + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3085,7 +3088,6 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR return 0; } - int32_t tSerializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 7cfec917b1..a574d802f9 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -119,10 +119,10 @@ _OVER: } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; - SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; + SMsgHandle * pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; SMgmtWrapper *pWrapper = pHandle->pNdWrapper; if (msgType == TDMT_DND_SERVER_STATUS) { @@ -443,7 +443,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); + void * pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -523,4 +523,4 @@ SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) { .pWrapper = pWrapper, }; return msgCb; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 42951beca2..33af040915 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -29,6 +29,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); +int32_t mndConvertRSmaTask(const char* ast, int8_t triggerType, int64_t watermark, char** pStr, int32_t* pLen); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 826a73afc6..ec4be8cba3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -24,20 +24,20 @@ #include "version.h" typedef struct { - uint32_t id; - int8_t connType; - char user[TSDB_USER_LEN]; - char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc - int64_t appStartTimeMs; // app start time - int32_t pid; // pid of app that invokes taosc - uint32_t ip; - uint16_t port; - int8_t killed; - int64_t loginTimeMs; - int64_t lastAccessTimeMs; - uint64_t killId; - int32_t numOfQueries; - SArray *pQueries; //SArray + uint32_t id; + int8_t connType; + char user[TSDB_USER_LEN]; + char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc + int64_t appStartTimeMs; // app start time + int32_t pid; // pid of app that invokes taosc + uint32_t ip; + uint16_t port; + int8_t killed; + int64_t loginTimeMs; + int64_t lastAccessTimeMs; + uint64_t killId; + int32_t numOfQueries; + SArray * pQueries; // SArray } SConnObj; static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port, @@ -45,7 +45,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); -static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); +static void * mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq); static int32_t mndProcessConnectReq(SNodeMsg *pReq); @@ -71,9 +71,9 @@ int32_t mndInitProfile(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_KILL_QUERY, mndProcessKillQueryReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_CONN, mndProcessKillConnReq); -// mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); + // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); -// mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); + // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery); return 0; @@ -91,7 +91,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - char connStr[255] = {0}; + char connStr[255] = {0}; int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app); int32_t connId = mndGenerateUid(connStr, len); if (startTime == 0) startTime = taosGetTimestampMs(); @@ -174,10 +174,10 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { } static int32_t mndProcessConnectReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; - SUserObj *pUser = NULL; - SDbObj *pDb = NULL; - SConnObj *pConn = NULL; + SMnode * pMnode = pReq->pNode; + SUserObj * pUser = NULL; + SDbObj * pDb = NULL; + SConnObj * pConn = NULL; int32_t code = -1; SConnectReq connReq = {0}; char ip[30] = {0}; @@ -194,6 +194,11 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) { mError("user:%s, failed to login while acquire user since %s", pReq->user, terrstr()); goto CONN_OVER; } + if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { + mError("user:%s, failed to auth while acquire user\n %s \r\n %s", pReq->user, connReq.passwd, pUser->pass); + code = TSDB_CODE_RPC_AUTH_FAILURE; + goto CONN_OVER; + } if (connReq.db[0]) { char db[TSDB_DB_FNAME_LEN]; @@ -253,8 +258,8 @@ static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) { pConn->pQueries = pBasic->queryDesc; pBasic->queryDesc = NULL; - - pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0; + + pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0; return TSDB_CODE_SUCCESS; } @@ -324,9 +329,10 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { return NULL; } -static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, SClientHbBatchRsp *pBatchRsp) { +static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, + SClientHbBatchRsp *pBatchRsp) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; + SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; if (pHbReq->query) { SQueryHbReqBasic *pBasic = pHbReq->query; @@ -335,8 +341,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb rpcGetConnInfo(pMsg->handle, &connInfo); SConnObj *pConn = mndAcquireConn(pMnode, pBasic->connId); - if (pConn == NULL) { - pConn = mndCreateConn(pMnode, connInfo.user, CONN_TYPE__QUERY, connInfo.clientIp, connInfo.clientPort, pBasic->pid, pBasic->app, 0); + if (pConn == NULL) { + pConn = mndCreateConn(pMnode, connInfo.user, CONN_TYPE__QUERY, connInfo.clientIp, connInfo.clientPort, + pBasic->pid, pBasic->app, 0); if (pConn == NULL) { mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr()); return -1; @@ -345,7 +352,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } } else if (pConn->killed) { mError("user:%s, conn:%u is already killed", connInfo.user, pConn->id); - mndReleaseConn(pMnode, pConn); + mndReleaseConn(pMnode, pConn); terrno = TSDB_CODE_MND_INVALID_CONNECTION; return -1; } @@ -369,8 +376,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } rspBasic->connId = pConn->id; - rspBasic->totalDnodes = 1; //TODO - rspBasic->onlineDnodes = 1; //TODO + rspBasic->totalDnodes = 1; // TODO + rspBasic->onlineDnodes = 1; // TODO mndGetMnodeEpSet(pMnode, &rspBasic->epSet); mndReleaseConn(pMnode, pConn); @@ -379,7 +386,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb int32_t kvNum = taosHashGetSize(pHbReq->info); if (NULL == pHbReq->info || kvNum <= 0) { - taosArrayPush(pBatchRsp->rsps, &hbRsp); + taosArrayPush(pBatchRsp->rsps, &hbRsp); return TSDB_CODE_SUCCESS; } @@ -396,7 +403,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb switch (kv->key) { case HEARTBEAT_KEY_DBINFO: { - void *rspMsg = NULL; + void * rspMsg = NULL; int32_t rspLen = 0; mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { @@ -406,7 +413,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb break; } case HEARTBEAT_KEY_STBINFO: { - void *rspMsg = NULL; + void * rspMsg = NULL; int32_t rspLen = 0; mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { @@ -457,7 +464,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq); int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp); - void *buf = rpcMallocCont(tlen); + void * buf = rpcMallocCont(tlen); tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps); @@ -479,7 +486,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { } static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; + SMnode * pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -513,7 +520,7 @@ static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) { } static int32_t mndProcessKillConnReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; + SMnode * pMnode = pReq->pNode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); @@ -545,11 +552,11 @@ static int32_t mndProcessKillConnReq(SNodeMsg *pReq) { } static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pNode; + SMnode * pMnode = pReq->pNode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; - char *pWrite; + char * pWrite; char ipStr[TSDB_IPv4ADDR_LEN + 6]; if (pShow->pIter == NULL) { @@ -604,8 +611,8 @@ static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int } static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->pNode; - int32_t numOfRows = 0; + SMnode *pMnode = pReq->pNode; + int32_t numOfRows = 0; #if 0 SConnObj *pConn = NULL; int32_t cols = 0; @@ -703,7 +710,7 @@ static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, i } pShow->numOfRows += numOfRows; -#endif +#endif return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index b3583af1dc..73583058f1 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -34,6 +34,54 @@ extern bool tsStreamSchedV; +int32_t mndConvertRSmaTask(const char* ast, int8_t triggerType, int64_t watermark, char** pStr, int32_t* pLen) { + SNode* pAst = NULL; + SQueryPlan* pPlan = NULL; + terrno = TSDB_CODE_SUCCESS; + + if (nodesStringToNode(ast, &pAst) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + + SPlanContext cxt = { + .pAstRoot = pAst, + .topicQuery = false, + .streamQuery = true, + .rSmaQuery = true, + .triggerType = triggerType, + .watermark = watermark, + }; + if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + + int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); + if (levelNum != 1) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0); + + int32_t opNum = LIST_LENGTH(inner->pNodeList); + if (opNum != 1) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + + SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); + if (qSubPlanToString(plan, pStr, pLen) < 0) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + goto END; + } + +END: + if (pAst) nodesDestroyNode(pAst); + if (pPlan) nodesDestroyNode(pPlan); + return terrno; +} + int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) { SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 2c3be2135b..9c8e0298aa 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -30,8 +30,15 @@ int32_t MndTestProfile::connId; TEST_F(MndTestProfile, 01_ConnectMsg) { SConnectReq connectReq = {0}; connectReq.pid = 1234; + + char passwd[] = "taosdata"; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt); + strcpy(connectReq.app, "mnode_test_profile"); strcpy(connectReq.db, ""); + strcpy(connectReq.user, "root"); + strcpy(connectReq.passwd, secretEncrypt); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); void* pReq = rpcMallocCont(contLen); @@ -58,10 +65,16 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { } TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) { + char passwd[] = "taosdata"; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt); + SConnectReq connectReq = {0}; connectReq.pid = 1234; strcpy(connectReq.app, "mnode_test_profile"); strcpy(connectReq.db, "invalid_db"); + strcpy(connectReq.user, "root"); + strcpy(connectReq.passwd, secretEncrypt); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index 201a42e3ef..2b0eaa0bec 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -54,10 +54,16 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { } TEST_F(MndTestShow, 03_ShowMsg_Conn) { + char passwd[] = "taosdata"; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt); + SConnectReq connectReq = {0}; connectReq.pid = 1234; strcpy(connectReq.app, "mnode_test_show"); strcpy(connectReq.db, ""); + strcpy(connectReq.user, "root"); + strcpy(connectReq.passwd, secretEncrypt); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); void* pReq = rpcMallocCont(contLen); @@ -74,4 +80,4 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { TEST_F(MndTestShow, 04_ShowMsg_Cluster) { test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "cluster", ""); EXPECT_EQ(test.GetShowRows(), 1); -} \ No newline at end of file +} diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 834d11fc20..2b52d333da 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -109,8 +109,7 @@ int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); -int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); -SArray *tqRetrieveDataBlock(STqReadHandle *pHandle); +int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows); // need to reposition diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index cf62ea8714..02ce6c4aad 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -82,16 +82,7 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { return false; } -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { - // currently only rows are used - - pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList); - pBlockInfo->rows = pHandle->pBlock->numOfRows; - // pBlockInfo->uid = pHandle->pBlock->uid; // the uid can not be assigned to pBlockData. - return 0; -} - -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { +int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows) { /*int32_t sversion = pHandle->pBlock->sversion;*/ // TODO set to real sversion int32_t sversion = 0; @@ -112,7 +103,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { STSchema* pTschema = pHandle->pSchema; SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; - int32_t numOfRows = pHandle->pBlock->numOfRows; + *pNumOfRows = pHandle->pBlock->numOfRows; /*int32_t numOfCols = pHandle->pSchema->numOfCols;*/ int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); @@ -120,10 +111,11 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { colNumNeed = pSchemaWrapper->nCols; } - SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); - if (pArray == NULL) { - return NULL; + *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); + if (*ppCols == NULL) { + return -1; } + int32_t colMeta = 0; int32_t colNeed = 0; while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { @@ -136,21 +128,24 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { colNeed++; } else { SColumnInfoData colInfo = {0}; - /*int sz = numOfRows * pColSchema->bytes;*/ colInfo.info.bytes = pColSchema->bytes; colInfo.info.colId = pColSchema->colId; colInfo.info.type = pColSchema->type; - if (colInfoDataEnsureCapacity(&colInfo, 0, numOfRows) < 0) { - taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); - return NULL; + if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + goto FAIL; } - taosArrayPush(pArray, &colInfo); + taosArrayPush(*ppCols, &colInfo); colMeta++; colNeed++; } } + int32_t colActual = taosArrayGetSize(*ppCols); + + // TODO in stream shuffle case, fetch groupId + *pGroupId = 0; + STSRowIter iter = {0}; tdSTSRowIterInit(&iter, pTschema); STSRow* row; @@ -159,22 +154,22 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block - int32_t colTot = taosArrayGetSize(pArray); - for (int32_t i = 0; i < colTot; i++) { - SColumnInfoData* pColData = taosArrayGet(pArray, i); + for (int32_t i = 0; i < colActual; i++) { + SColumnInfoData* pColData = taosArrayGet(*ppCols, i); SCellVal sVal = {0}; if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } - /*if (colDataAppend(pColData, curRow, sVal.val, false) < 0) {*/ if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { - taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); - return NULL; + goto FAIL; } } curRow++; } - return pArray; + return 0; +FAIL: + taosArrayDestroy(*ppCols); + return -1; } void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f0d3d95b6a..b054bbfcb6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -30,12 +30,11 @@ #include "query.h" #include "tcompare.h" #include "thash.h" -#include "vnode.h" #include "ttypes.h" +#include "vnode.h" #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) -#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) - +#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { @@ -90,7 +89,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord } int64_t key = tw->skey, interval = pInterval->interval; - //convert key to second + // convert key to second key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; if (pInterval->intervalUnit == 'y') { @@ -98,7 +97,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord } struct tm tm; - time_t t = (time_t)key; + time_t t = (time_t)key; taosLocalTime(&t, &tm); int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); @@ -125,8 +124,8 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn // todo handle the time range case TSKEY sk = INT64_MIN; TSKEY ek = INT64_MAX; -// TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + // TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + // TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); if (true) { getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey, &w); @@ -136,7 +135,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn return true; } - while(1) { // todo handle the desc order scan case + while (1) { // todo handle the desc order scan case getNextTimeWindow(pInterval, &w, TSDB_ORDER_ASC); if (w.skey > pBlockInfo->window.ekey) { break; @@ -148,31 +147,31 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } } } else { -// getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); -// assert(w.skey <= pBlockInfo->window.ekey); -// -// if (w.skey > pBlockInfo->window.skey) { -// return true; -// } -// -// while(1) { -// getNextTimeWindow(pQueryAttr, &w); -// if (w.ekey < pBlockInfo->window.skey) { -// break; -// } -// -// assert(w.skey < pBlockInfo->window.skey); -// if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { -// return true; -// } -// } + // getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); + // assert(w.skey <= pBlockInfo->window.ekey); + // + // if (w.skey > pBlockInfo->window.skey) { + // return true; + // } + // + // while(1) { + // getNextTimeWindow(pQueryAttr, &w); + // if (w.ekey < pBlockInfo->window.skey) { + // break; + // } + // + // assert(w.skey < pBlockInfo->window.skey); + // if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { + // return true; + // } + // } } return false; } int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableScanInfo* pInfo = pOperator->info; STaskCostInfo* pCost = &pTaskInfo->cost; @@ -189,13 +188,13 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, taosMemoryFreeClear(pBlock->pBlockAgg); if (*status == FUNC_DATA_REQUIRED_FILTEROUT) { - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { @@ -218,12 +217,12 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, } return TSDB_CODE_SUCCESS; - } else { // failed to load the block sma data, data block statistics does not exist, load data block instead + } else { // failed to load the block sma data, data block statistics does not exist, load data block instead *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } - ASSERT (*status == FUNC_DATA_REQUIRED_DATA_LOAD); + ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); // todo filter data block according to the block sma data firstly #if 0 @@ -249,8 +248,8 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, doFilter(pTableScanInfo->pFilterNode, pBlock); if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, - pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } return TSDB_CODE_SUCCESS; @@ -348,9 +347,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; -// if (pResultRowInfo->size > 0) { -// pResultRowInfo->curPos = 0; -// } + // if (pResultRowInfo->size > 0) { + // pResultRowInfo->curPos = 0; + // } qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); @@ -367,7 +366,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); if (pResultRowInfo->size > 0) { -// pResultRowInfo->curPos = pResultRowInfo->size - 1; + // pResultRowInfo->curPos = pResultRowInfo->size - 1; } p = doTableScanImpl(pOperator, newgroup); @@ -376,9 +375,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { return p; } -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag, - int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, - SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, + int32_t dataLoadFlag, int32_t repeatTime, int32_t reverseTime, + SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, + SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); @@ -391,26 +391,26 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, return NULL; } - pInfo->interval = *pInterval; - pInfo->sampleRatio = sampleRatio; - pInfo->dataBlockLoadFlag= dataLoadFlag; - pInfo->pResBlock = pResBlock; - pInfo->pFilterNode = pCondition; - pInfo->dataReader = pTsdbReadHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = reverseTime; - pInfo->order = order; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColMatchInfo; - pOperator->name = "TableScanOperator"; + pInfo->interval = *pInterval; + pInfo->sampleRatio = sampleRatio; + pInfo->dataBlockLoadFlag = dataLoadFlag; + pInfo->pResBlock = pResBlock; + pInfo->pFilterNode = pCondition; + pInfo->dataReader = pTsdbReadHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->order = order; + pInfo->current = 0; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; + pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->getNextFn = doTableScan; + pOperator->pTaskInfo = pTaskInfo; static int32_t cost = 0; pOperator->cost.openCost = ++cost; @@ -456,67 +456,67 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { ++numRowSteps; } - tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader); + tableBlockDist.numOfRowsInMemTable = (int32_t)tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader); SSDataBlock* pBlock = pTableScanInfo->pResBlock; - pBlock->info.rows = 1; + pBlock->info.rows = 1; pBlock->info.numOfCols = 1; -// SBufferWriter bw = tbufInitWriter(NULL, false); -// blockDistInfoToBinary(&tableBlockDist, &bw); + // SBufferWriter bw = tbufInitWriter(NULL, false); + // blockDistInfoToBinary(&tableBlockDist, &bw); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); -// int32_t len = (int32_t) tbufTell(&bw); -// pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); -// *(int32_t*) pColInfo->pData = len; -// memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); -// -// tbufCloseWriter(&bw); + // int32_t len = (int32_t) tbufTell(&bw); + // pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); + // *(int32_t*) pColInfo->pData = len; + // memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); + // + // tbufCloseWriter(&bw); -// SArray* g = GET_TABLEGROUP(pOperator->, 0); -// pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); + // SArray* g = GET_TABLEGROUP(pOperator->, 0); + // pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); pOperator->status = OP_EXEC_DONE; return pBlock; } SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo) { - STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } - pInfo->dataReader = dataReader; -// pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + pInfo->dataReader = dataReader; + // pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_BINARY; + infoData.info.type = TSDB_DATA_TYPE_BINARY; infoData.info.bytes = 1024; infoData.info.colId = 0; -// taosArrayPush(pInfo->block.pDataBlock, &infoData); + // taosArrayPush(pInfo->block.pDataBlock, &infoData); - pOperator->name = "DataBlockInfoScanOperator"; + pOperator->name = "DataBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doBlockInfoScan; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; return pOperator; - _error: +_error: taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); return NULL; @@ -558,29 +558,42 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) blockDataCleanup(pInfo->pRes); while (tqNextDataBlock(pInfo->readerHandle)) { - pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - terrno = pTaskInfo->code; - pOperator->status = OP_EXEC_DONE; + SArray* pCols = NULL; + uint64_t groupId; + int32_t numOfRows; + int32_t code = tqRetrieveDataBlock(&pCols, pInfo->readerHandle, &groupId, &numOfRows); + + if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { + pTaskInfo->code = code; return NULL; } - if (pBlockInfo->rows == 0) { - break; - } - - SArray* pCols = tqRetrieveDataBlock(pInfo->readerHandle); + pInfo->pRes->info.groupId = groupId; + pInfo->pRes->info.rows = numOfRows; int32_t numOfCols = pInfo->pRes->info.numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pCols, i); - SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); + SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { continue; } - ASSERT(pColMatchInfo->colId == p->info.colId); - taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, p); + bool colExists = false; + for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) { + SColumnInfoData* pResCol = taosArrayGet(pCols, j); + if (pResCol->info.colId == pColMatchInfo->colId) { + taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); + colExists = true; + break; + } + } + + // the required column does not exists in submit block, let's set it to be all null value + if (!colExists) { + SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); + colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows); + colDataAppendNNULL(pDst, 0, pBlockInfo->rows); + } } if (pInfo->pRes->pDataBlock == NULL) { @@ -605,7 +618,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) } } -SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, + SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -618,7 +632,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* int32_t numOfOutput = taosArrayGetSize(pColList); SArray* pColIds = taosArrayInit(4, sizeof(int16_t)); - for(int32_t i = 0; i < numOfOutput; ++i) { + for (int32_t i = 0; i < numOfOutput; ++i) { int16_t* id = taosArrayGet(pColList, i); taosArrayPush(pColIds, id); } @@ -644,16 +658,16 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pInfo->readerHandle = streamReadHandle; pInfo->pRes = pResBlock; - pOperator->name = "StreamBlockScanOperator"; + pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doStreamBlockScan; - pOperator->closeFn = operatorDummyCloseFn; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doStreamBlockScan; + pOperator->closeFn = operatorDummyCloseFn; + pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -733,9 +747,9 @@ static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->handle = htobe64(pRsp->handle); - pRsp->compLen = htonl(pRsp->compLen); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->handle = htobe64(pRsp->handle); + pRsp->compLen = htonl(pRsp->compLen); } else { operator->pTaskInfo->code = code; } @@ -777,7 +791,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { if (rowRes[j] == 0) { continue; } - + colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); numOfRow += 1; } @@ -828,7 +842,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i); int64_t tmp = 0; char t[10] = {0}; - STR_TO_VARSTR(t, "_"); //TODO + STR_TO_VARSTR(t, "_"); // TODO if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { colDataAppend(pColInfoData, numOfRows, t, false); } else { @@ -926,12 +940,12 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB return NULL; } - pInfo->accountId = accountId; + pInfo->accountId = accountId; pInfo->showRewrite = showRewrite; - pInfo->pRes = pResBlock; - pInfo->capacity = 4096; - pInfo->pCondition = pCondition; - pInfo->scanCols = colList; + pInfo->pRes = pResBlock; + pInfo->capacity = 4096; + pInfo->pCondition = pCondition; + pInfo->scanCols = colList; // TODO remove it int32_t tableType = 0; @@ -986,9 +1000,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB tableType = TSDB_MGMT_TABLE_CONNS; } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, tListLen(pName->tname)) == 0) { tableType = TSDB_MGMT_TABLE_QUERIES; - } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, tListLen(pName->tname)) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, tListLen(pName->tname)) == 0) { tableType = TSDB_MGMT_TABLE_VNODES; - }else { + } else { ASSERT(0); } @@ -1025,15 +1039,15 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB #endif } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->getNextFn = doSysTableScan; - pOperator->closeFn = destroySysScanOperator; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->getNextFn = doSysTableScan; + pOperator->closeFn = destroySysScanOperator; + pOperator->pTaskInfo = pTaskInfo; return pOperator; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index b19757bd05..0559792740 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -716,7 +716,8 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false"); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { - int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), varDataVal(input)); + int32_t len = MIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE); + len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input)); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { //not support diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 1b8cbb0f04..e18538cf27 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -158,6 +158,7 @@ typedef struct { char secured : 2; char spi : 2; + char user[TSDB_UNI_LEN]; uint64_t ahandle; // ahandle assigned by client uint32_t code; // del later uint32_t msgType; @@ -186,23 +187,23 @@ typedef enum { Normal, Quit, Release, Register } STransMsgType; typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } ConnStatus; #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) -#define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) +#define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) -#define RPC_MSG_OVERHEAD (sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) -#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) +#define RPC_MSG_OVERHEAD (sizeof(SRpcHead) + sizeof(SRpcDigest)) +#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) +#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) #define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) -#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) -#define rpcIsReq(type) (type & 1U) +#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) +#define rpcIsReq(type) (type & 1U) #define TRANS_RESERVE_SIZE (sizeof(STranConnCtx)) -#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead)) -#define transHeadFromCont(cont) ((STransMsgHead*)((char*)cont - sizeof(STransMsgHead))) -#define transContFromHead(msg) (msg + sizeof(STransMsgHead)) +#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead)) +#define transHeadFromCont(cont) ((STransMsgHead*)((char*)cont - sizeof(STransMsgHead))) +#define transContFromHead(msg) (msg + sizeof(STransMsgHead)) #define transMsgLenFromCont(contLen) (contLen + sizeof(STransMsgHead)) -#define transContLenFromMsg(msgLen) (msgLen - sizeof(STransMsgHead)); -#define transIsReq(type) (type & 1U) +#define transContLenFromMsg(msgLen) (msgLen - sizeof(STransMsgHead)); +#define transIsReq(type) (type & 1U) int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8eb1a3ee7d..b81310b90b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -614,35 +614,16 @@ void cliSend(SCliConn* pConn) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; } - STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; - int msgLen = transMsgLenFromCont(pMsg->contLen); - if (!pConn->secured) { - char* buf = taosMemoryCalloc(1, msgLen + sizeof(STransUserMsg)); - memcpy(buf, (char*)pHead, msgLen); - - STransUserMsg* uMsg = (STransUserMsg*)(buf + msgLen); - memcpy(uMsg->user, pTransInst->user, tListLen(uMsg->user)); - memcpy(uMsg->secret, pTransInst->secret, tListLen(uMsg->secret)); - - // to avoid mem leak - destroyUserdata(pMsg); - - pMsg->pCont = (char*)buf + sizeof(STransMsgHead); - pMsg->contLen = msgLen + sizeof(STransUserMsg) - sizeof(STransMsgHead); - - pHead = (STransMsgHead*)buf; - pHead->secured = 1; - msgLen += sizeof(STransUserMsg); - } - + STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; pHead->persist = REQUEST_PERSIS_HANDLE(pMsg) ? 1 : 0; pHead->msgType = pMsg->msgType; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; + memcpy(pHead->user, pTransInst->user, strlen(pTransInst->user)); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 158d599bdf..ec66f3e8df 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -46,7 +46,6 @@ typedef struct SSrvConn { struct sockaddr_in addr; struct sockaddr_in locaddr; - char secured; int spi; char info[64]; char user[TSDB_UNI_LEN]; // user ID for the link @@ -104,6 +103,13 @@ static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); static void uvShutDownCb(uv_shutdown_t* req, int status); + +/* + * time-consuming task throwed into BG work thread + */ +static void uvWorkDoTask(uv_work_t* req); +static void uvWorkAfterTask(uv_work_t* req, int status); + static void uvWalkCb(uv_handle_t* handle, void* arg); static void uvFreeCb(uv_handle_t* handle); @@ -181,16 +187,16 @@ static void uvHandleReq(SSrvConn* pConn) { uint32_t msgLen = pBuf->len; STransMsgHead* pHead = (STransMsgHead*)msg; - if (pHead->secured == 1) { - STransUserMsg* uMsg = (STransUserMsg*)((char*)msg + msgLen - sizeof(STransUserMsg)); - memcpy(pConn->user, uMsg->user, tListLen(uMsg->user)); - memcpy(pConn->secret, uMsg->secret, tListLen(uMsg->secret)); - } pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); - if (pHead->secured == 1) { - pHead->msgLen -= sizeof(STransUserMsg); - } + memcpy(pConn->user, pHead->user, strlen(pHead->user)); + + // TODO(dengyihao): time-consuming task throwed into BG Thread + // uv_work_t* wreq = taosMemoryMalloc(sizeof(uv_work_t)); + // wreq->data = pConn; + // uv_read_stop((uv_stream_t*)pConn->pTcp); + // transRefSrvHandle(pConn); + // uv_queue_work(((SWorkThrdObj*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); CONN_SHOULD_RELEASE(pConn, pHead); @@ -344,12 +350,6 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); pHead->ahandle = (uint64_t)pMsg->ahandle; - // pHead->secured = pMsg->code == 0 ? 1 : 0; // - if (!pConn->secured) { - pConn->secured = pMsg->code == 0 ? 1 : 0; - } - pHead->secured = pConn->secured; - if (pConn->status == ConnNormal) { pHead->msgType = pConn->inType + 1; } else { @@ -464,6 +464,24 @@ static void uvShutDownCb(uv_shutdown_t* req, int status) { taosMemoryFree(req); } +static void uvWorkDoTask(uv_work_t* req) { + // doing time-consumeing task + // only auth conn currently, add more func later + tTrace("server conn %p start to be processed in BG Thread", req->data); + return; +} + +static void uvWorkAfterTask(uv_work_t* req, int status) { + if (status != 0) { + tTrace("server conn %p failed to processed ", req->data); + } + // Done time-consumeing task + // add more func later + // this func called in main loop + tTrace("server conn %p already processed ", req->data); + taosMemoryFree(req); +} + void uvOnAcceptCb(uv_stream_t* stream, int status) { if (status == -1) { return; diff --git a/tests/script/runAllSimCases.sh b/tests/script/runAllSimCases.sh index 279bc8363e..97bebd9eb8 100755 --- a/tests/script/runAllSimCases.sh +++ b/tests/script/runAllSimCases.sh @@ -1,4 +1,4 @@ -!/bin/bash +#!/bin/bash ################################################## # diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 2b3e9963f0..50b4a0800f 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -340,7 +340,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); - tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); tmq_conf_destroy(conf); return tmq; @@ -367,7 +367,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1); if (tmqmessage) { /*msg_process(tmqmessage);*/ - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); } @@ -400,7 +400,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog if (0 != g_stConfInfo.showMsgFlag) { /*msg_process(tmqmessage);*/ } - tmq_message_destroy(tmqmessage); + taos_free_result(tmqmessage); } else { break; } diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 5546e514cc..d3bed600dd 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -31,46 +31,46 @@ #define NC "\033[0m" #define min(a, b) (((a) < (b)) ? (a) : (b)) -#define MAX_SQL_STR_LEN (1024 * 1024) -#define MAX_ROW_STR_LEN (16 * 1024) -#define MAX_CONSUMER_THREAD_CNT (16) +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_CONSUMER_THREAD_CNT (16) typedef struct { - TdThread thread; - int32_t consumerId; + TdThread thread; + int32_t consumerId; - int32_t ifCheckData; - int64_t expectMsgCnt; - - int64_t consumeMsgCnt; - int32_t checkresult; + int32_t ifCheckData; + int64_t expectMsgCnt; - char topicString[1024]; - char keyString[1024]; + int64_t consumeMsgCnt; + int32_t checkresult; - int32_t numOfTopic; - char topics[32][64]; + char topicString[1024]; + char keyString[1024]; - int32_t numOfKey; - char key[32][64]; - char value[32][64]; + int32_t numOfTopic; + char topics[32][64]; + + int32_t numOfKey; + char key[32][64]; + char value[32][64]; tmq_t* tmq; tmq_list_t* topicList; - + } SThreadInfo; typedef struct { // input from argvs - char dbName[32]; - int32_t showMsgFlag; - int32_t consumeDelay; // unit s - int32_t numOfThread; - SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; + char dbName[32]; + int32_t showMsgFlag; + int32_t consumeDelay; // unit s + int32_t numOfThread; + SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; } SConfInfo; static SConfInfo g_stConfInfo; -TdFilePtr g_fp = NULL; +TdFilePtr g_fp = NULL; // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -95,7 +95,7 @@ void initLogFile() { TdFilePtr pFile = taosOpenFile("./tmqlog.txt", TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt"); - exit -1; + exit - 1; }; g_fp = pFile; @@ -103,27 +103,27 @@ void initLogFile() { struct tm tm = *taosLocalTime(&tTime, NULL); taosFprintfFile(pFile, "###################################################################\n"); - taosFprintfFile(pFile, "# configDir: %s\n", configDir); - taosFprintfFile(pFile, "# dbName: %s\n", g_stConfInfo.dbName); - taosFprintfFile(pFile, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); - taosFprintfFile(pFile, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + taosFprintfFile(pFile, "# configDir: %s\n", configDir); + taosFprintfFile(pFile, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(pFile, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); + taosFprintfFile(pFile, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosFprintfFile(pFile, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - taosFprintfFile(pFile, " Topics: "); - for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfTopic; i++) { - taosFprintfFile(pFile, "%s, ", g_stConfInfo.stThreads[i].topics[i]); + taosFprintfFile(pFile, " Topics: "); + for (int i = 0; i < g_stConfInfo.stThreads[i].numOfTopic; i++) { + taosFprintfFile(pFile, "%s, ", g_stConfInfo.stThreads[i].topics[i]); } - taosFprintfFile(pFile, "\n"); + taosFprintfFile(pFile, "\n"); taosFprintfFile(pFile, " Key: "); - for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfKey; i++) { - taosFprintfFile(pFile, "%s:%s, ", g_stConfInfo.stThreads[i].key[i], g_stConfInfo.stThreads[i].value[i]); + for (int i = 0; i < g_stConfInfo.stThreads[i].numOfKey; i++) { + taosFprintfFile(pFile, "%s:%s, ", g_stConfInfo.stThreads[i].key[i], g_stConfInfo.stThreads[i].value[i]); } taosFprintfFile(pFile, "\n"); } - + taosFprintfFile(pFile, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); taosFprintfFile(pFile, "###################################################################\n"); } @@ -180,23 +180,23 @@ void ltrim(char* str) { // return str; } -static int running = 1; +static int running = 1; static void msg_process(TAOS_RES* msg, int32_t msgIndex, int32_t threadLable) { char buf[1024]; - //printf("topic: %s\n", tmq_get_topic_name(msg)); - //printf("vg:%d\n", tmq_get_vgroup_id(msg)); - taosFprintfFile(g_fp, "msg index:%d, threadLable: %d\n", msgIndex, threadLable); - taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); - + // printf("topic: %s\n", tmq_get_topic_name(msg)); + // printf("vg:%d\n", tmq_get_vgroup_id(msg)); + taosFprintfFile(g_fp, "msg index:%d, threadLable: %d\n", msgIndex, threadLable); + taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); + while (1) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); - //taos_print_row(buf, row, fields, numOfFields); - //printf("%s\n", buf); - //taosFprintfFile(g_fp, "%s\n", buf); + // taos_print_row(buf, row, fields, numOfFields); + // printf("%s\n", buf); + // taosFprintfFile(g_fp, "%s\n", buf); } } @@ -213,7 +213,7 @@ int queryDB(TAOS* taos, char* command) { return 0; } -void build_consumer(SThreadInfo *pInfo) { +void build_consumer(SThreadInfo* pInfo) { char sqlStr[1024] = {0}; TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); @@ -233,11 +233,11 @@ void build_consumer(SThreadInfo *pInfo) { for (int32_t i = 0; i < pInfo->numOfKey; i++) { tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); } - pInfo->tmq = tmq_consumer_new(pConn, conf, NULL, 0); + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); return; } -void build_topic_list(SThreadInfo *pInfo) { +void build_topic_list(SThreadInfo* pInfo) { pInfo->topicList = tmq_list_new(); // tmq_list_append(topic_list, "test_stb_topic_1"); for (int32_t i = 0; i < pInfo->numOfTopic; i++) { @@ -246,48 +246,45 @@ void build_topic_list(SThreadInfo *pInfo) { return; } -int32_t saveConsumeResult(SThreadInfo *pInfo) { +int32_t saveConsumeResult(SThreadInfo* pInfo) { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %d)", - g_stConfInfo.dbName, - pInfo->consumerId, - pInfo->consumeMsgCnt, - pInfo->checkresult); - + sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %d)", g_stConfInfo.dbName, + pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->checkresult); + TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); exit(-1); } - + taos_free_result(pRes); return 0; } -void loop_consume(SThreadInfo *pInfo) { +void loop_consume(SThreadInfo* pInfo) { tmq_resp_err_t err; - + int64_t totalMsgs = 0; - //int64_t totalRows = 0; + // int64_t totalRows = 0; while (running) { TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); - if (tmqMsg) { + if (tmqMsg) { if (0 != g_stConfInfo.showMsgFlag) { msg_process(tmqMsg, totalMsgs, 0); } - - tmq_message_destroy(tmqMsg); - + + taos_free_result(tmqMsg); + totalMsgs++; - + if (totalMsgs >= pInfo->expectMsgCnt) { break; } @@ -295,7 +292,7 @@ void loop_consume(SThreadInfo *pInfo) { break; } } - + err = tmq_consumer_close(pInfo->tmq); if (err) { printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); @@ -303,35 +300,34 @@ void loop_consume(SThreadInfo *pInfo) { } pInfo->consumeMsgCnt = totalMsgs; - } -void *consumeThreadFunc(void *param) { +void* consumeThreadFunc(void* param) { int32_t totalMsgs = 0; - SThreadInfo *pInfo = (SThreadInfo *)param; + SThreadInfo* pInfo = (SThreadInfo*)param; build_consumer(pInfo); build_topic_list(pInfo); - if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)){ + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { return NULL; } - + tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); if (err) { printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - + loop_consume(pInfo); err = tmq_unsubscribe(pInfo->tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - pInfo->consumeMsgCnt = -1; + pInfo->consumeMsgCnt = -1; return NULL; - } - + } + // save consume result into consumeresult table saveConsumeResult(pInfo); @@ -343,7 +339,7 @@ void parseConsumeInfo() { const char delim[2] = ","; const char ch = ':'; - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { token = strtok(g_stConfInfo.stThreads[i].topicString, delim); while (token != NULL) { // printf("%s\n", token ); @@ -351,10 +347,10 @@ void parseConsumeInfo() { ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); g_stConfInfo.stThreads[i].numOfTopic++; - + token = strtok(NULL, delim); } - + token = strtok(g_stConfInfo.stThreads[i].keyString, delim); while (token != NULL) { // printf("%s\n", token ); @@ -368,7 +364,7 @@ void parseConsumeInfo() { // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.stThreads[i].numOfKey++; } - + token = strtok(NULL, delim); } } @@ -376,46 +372,47 @@ void parseConsumeInfo() { int32_t getConsumeInfo() { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - + sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.dbName); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); exit(-1); - } - - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(pRes); - TAOS_FIELD* fields = taos_fetch_fields(pRes); - - // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int - + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(pRes); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + + // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, + // ifcheckdata int + int32_t numOfThread = 0; while ((row = taos_fetch_row(pRes))) { - int32_t* lengths = taos_fetch_lengths(pRes); - - for (int i = 0; i < num_fields; ++i) { + int32_t* lengths = taos_fetch_lengths(pRes); + + for (int i = 0; i < num_fields; ++i) { if (row[i] == NULL || 0 == i) { continue; } - + if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t *)row[i]); + g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]); } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { - g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t *)row[i]); + g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t *)row[i]); + g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); } } - numOfThread ++; + numOfThread++; } g_stConfInfo.numOfThread = numOfThread; @@ -426,7 +423,6 @@ int32_t getConsumeInfo() { return 0; } - int main(int32_t argc, char* argv[]) { parseArgument(argc, argv); getConsumeInfo(); @@ -438,18 +434,19 @@ int main(int32_t argc, char* argv[]) { // pthread_create one thread to consume for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { - taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, (void *)(&(g_stConfInfo.stThreads[i]))); + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); } for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); } - //printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); - - taosFprintfFile(g_fp, "\n"); - taosCloseFile(&g_fp); - + // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); + + taosFprintfFile(g_fp, "\n"); + taosCloseFile(&g_fp); + return 0; }