diff --git a/include/client/taos.h b/include/client/taos.h index 17f97d3d3d..94a7884950 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -360,7 +360,8 @@ typedef enum tmq_res_t { TMQ_RES_INVALID = -1, TMQ_RES_DATA = 1, TMQ_RES_TABLE_META = 2, - TMQ_RES_METADATA = 3 + TMQ_RES_METADATA = 3, + TMQ_RES_RAWDATA = 4 } tmq_res_t; typedef struct tmq_topic_assignment { diff --git a/include/common/tcommon.h b/include/common/tcommon.h index c30f2ab4ec..bd5bdb927d 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -121,10 +121,11 @@ enum { TMQ_MSG_TYPE__POLL_DATA_META_RSP, TMQ_MSG_TYPE__WALINFO_RSP, TMQ_MSG_TYPE__POLL_BATCH_META_RSP, + TMQ_MSG_TYPE__POLL_RAW_DATA_RSP, }; static char* tmqMsgTypeStr[] = { - "data", "meta", "ask ep", "meta data", "wal info", "batch meta" + "data", "meta", "ask ep", "meta data", "wal info", "batch meta", "raw data" }; enum { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6f78ec32a5..0c41c9f984 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2312,6 +2312,10 @@ typedef struct SSysTableSchema { int32_t tSerializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq); int32_t tDeserializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq); +#define RETRIEVE_TABLE_RSP_VERSION 0 +#define RETRIEVE_TABLE_RSP_TMQ_VERSION 1 +#define RETRIEVE_TABLE_RSP_TMQ_RAW_VERSION 2 + typedef struct { int64_t useconds; int8_t completed; // all results are returned to client @@ -4200,7 +4204,9 @@ typedef struct { STqOffsetVal reqOffset; int8_t enableReplay; int8_t sourceExcluded; + int8_t rawData; int8_t enableBatchMeta; + SHashObj *uidHash; // to find if uid is duplicated } SMqPollReq; int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); @@ -4274,13 +4280,21 @@ typedef struct { SArray* createTableLen; SArray* createTableReq; }; + struct{ + int32_t len; + void* rawData; + }; }; + void* data; //for free in client, only effected if type is data or metadata. raw data not effected + bool blockDataElementFree; // if true, free blockDataElement in blockData,(true in server, false in client) } SMqDataRsp; int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pObj); int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); +int32_t tDecodeMqRawDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); void tDeleteMqDataRsp(SMqDataRsp* pRsp); +void tDeleteMqRawDataRsp(SMqDataRsp* pRsp); int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); @@ -4530,6 +4544,7 @@ typedef struct { typedef struct { SArray* aSubmitTbData; // SArray + bool raw; } SSubmitReq2; typedef struct { @@ -4538,8 +4553,9 @@ typedef struct { char data[]; // SSubmitReq2 } SSubmitReq2Msg; +int32_t transformRawSSubmitTbData(void* data, int64_t suid, int64_t uid, int32_t sver); int32_t tEncodeSubmitReq(SEncoder* pCoder, const SSubmitReq2* pReq); -int32_t tDecodeSubmitReq(SDecoder* pCoder, SSubmitReq2* pReq); +int32_t tDecodeSubmitReq(SDecoder* pCoder, SSubmitReq2* pReq, SArray* rawList); void tDestroySubmitTbData(SSubmitTbData* pTbData, int32_t flag); void tDestroySubmitReq(SSubmitReq2* pReq, int32_t flag); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 95f522f504..f4bf5fafd4 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -180,8 +180,11 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int32_t msgBufLen, void* charsetCxt); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); +int32_t smlBuildOutputRaw(SQuery* handle, SHashObj* pVgHash); +int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data); int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields, int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw); +int32_t checkSchema(SSchema* pColSchema, int8_t* fields, char* errstr, int32_t errstrLen); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 464dffa937..397118411c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -1016,6 +1016,8 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TMQ_NO_TABLE_QUALIFIED TAOS_DEF_ERROR_CODE(0, 0x4015) #define TSDB_CODE_TMQ_NO_NEED_REBALANCE TAOS_DEF_ERROR_CODE(0, 0x4016) #define TSDB_CODE_TMQ_INVALID_STATUS TAOS_DEF_ERROR_CODE(0, 0x4017) +#define TSDB_CODE_TMQ_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x4018) +#define TSDB_CODE_TMQ_RAW_DATA_SPLIT TAOS_DEF_ERROR_CODE(0, 0x4019) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 2543a1f3ec..c9e0410341 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,6 +45,7 @@ enum { RES_TYPE__TMQ_META, RES_TYPE__TMQ_METADATA, RES_TYPE__TMQ_BATCH_META, + RES_TYPE__TMQ_RAWDATA, }; #define SHOW_VARIABLES_RESULT_COLS 5 @@ -55,6 +56,7 @@ enum { #define SHOW_VARIABLES_RESULT_FIELD5_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) #define TD_RES_QUERY(res) (*(int8_t*)(res) == RES_TYPE__QUERY) +#define TD_RES_TMQ_RAW(res) (*(int8_t*)(res) == RES_TYPE__TMQ_RAWDATA) #define TD_RES_TMQ(res) (*(int8_t*)(res) == RES_TYPE__TMQ) #define TD_RES_TMQ_META(res) (*(int8_t*)(res) == RES_TYPE__TMQ_META) #define TD_RES_TMQ_METADATA(res) (*(int8_t*)(res) == RES_TYPE__TMQ_METADATA) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 27e21d9a58..47414228ca 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -502,7 +502,7 @@ void taos_close(TAOS *taos) { } int taos_errno(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return terrno; } @@ -514,7 +514,7 @@ int taos_errno(TAOS_RES *res) { } const char *taos_errstr(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return (const char *)tstrerror(terrno); } @@ -554,6 +554,8 @@ void taos_free_result(TAOS_RES *res) { tDeleteMqMetaRsp(&pRsp->metaRsp); } else if (TD_RES_TMQ_BATCH_META(res)) { tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); + } else if (TD_RES_TMQ_RAW(res)) { + tDeleteMqRawDataRsp(&pRsp->dataRsp); } taosMemoryFree(pRsp); } @@ -572,7 +574,7 @@ void taos_kill_query(TAOS *taos) { } int taos_field_count(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -583,7 +585,7 @@ int taos_field_count(TAOS_RES *res) { int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); } TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { - if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (taos_num_fields(res) == 0 || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return NULL; } @@ -643,7 +645,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } else { tscError("invalid result passed to taos_fetch_row"); - terrno = TSDB_CODE_TSC_INTERNAL_ERROR; + terrno = TSDB_CODE_TMQ_INVALID_DATA; return NULL; } } @@ -764,7 +766,7 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD } int *taos_fetch_lengths(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return NULL; } @@ -773,7 +775,7 @@ int *taos_fetch_lengths(TAOS_RES *res) { } TAOS_ROW *taos_result_block(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -841,7 +843,7 @@ const char *taos_get_client_info() { return td_version; } // return int32_t int taos_affected_rows(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -853,7 +855,7 @@ int taos_affected_rows(TAOS_RES *res) { // return int64_t int64_t taos_affected_rows64(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -864,7 +866,7 @@ int64_t taos_affected_rows64(TAOS_RES *res) { } int taos_result_precision(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return TSDB_TIME_PRECISION_MILLI; } @@ -904,7 +906,7 @@ int taos_select_db(TAOS *taos, const char *db) { } void taos_stop_query(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return; } @@ -913,7 +915,7 @@ void taos_stop_query(TAOS_RES *res) { } bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return true; } SReqResultInfo *pResultInfo = tscGetCurResInfo(res); @@ -938,7 +940,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { } int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -973,7 +975,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { return 0; } else { tscError("taos_fetch_block_s invalid res type"); - return -1; + return TSDB_CODE_TMQ_INVALID_DATA; } } @@ -981,7 +983,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { *numOfRows = 0; *pData = NULL; - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -1018,7 +1020,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { } int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -1038,7 +1040,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows) { if (res == NULL || result == NULL || rows == NULL || *rows <= 0 || columnIndex < 0 || TD_RES_TMQ_META(res) || - TD_RES_TMQ_BATCH_META(res)) { + TD_RES_TMQ_RAW(res) || TD_RES_TMQ_BATCH_META(res)) { return TSDB_CODE_INVALID_PARA; } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 4ff50692b7..e99d5c2e83 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1844,9 +1844,10 @@ static void* getRawDataFromRes(void* pRetrieve) { } void* rawData = NULL; // deal with compatibility - if (*(int64_t*)pRetrieve == 0) { + if (*(int64_t*)pRetrieve == RETRIEVE_TABLE_RSP_VERSION) { rawData = ((SRetrieveTableRsp*)pRetrieve)->data; - } else if (*(int64_t*)pRetrieve == 1) { + } else if (*(int64_t*)pRetrieve == RETRIEVE_TABLE_RSP_TMQ_VERSION || + *(int64_t*)pRetrieve == RETRIEVE_TABLE_RSP_TMQ_RAW_VERSION) { rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data; } return rawData; @@ -1947,7 +1948,10 @@ static int32_t initRawCacheHash() { } static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW) { - if (rawData == NULL || pTableMeta == NULL || pSW == NULL) { + if (rawData == NULL || pSW == NULL){ + return false; + } + if (pTableMeta == NULL) { uError("invalid parameter in %s", __func__); return false; } @@ -1965,6 +1969,7 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe if (pSW->nCols != pTableMeta->tableInfo.numOfColumns) { return true; } + for (int i = 0; i < pSW->nCols; i++) { int j = 0; for (; j < pTableMeta->tableInfo.numOfColumns; j++) { @@ -1972,7 +1977,7 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe char* fieldName = pSW->pSchema[i].name; if (strcmp(pColSchema->name, fieldName) == 0) { - if (*fields != pColSchema->type || *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + if (checkSchema(pColSchema, fields, NULL, 0) != 0){ return true; } break; @@ -2069,7 +2074,7 @@ static int32_t processCacheMeta(SHashObj* pVgHash, SHashObj* pNameHash, SHashObj SVCreateTbReq* pCreateReqDst, SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName, STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry) { if (pVgHash == NULL || pNameHash == NULL || pMetaHash == NULL || pCatalog == NULL || conn == NULL || pName == NULL || - pMeta == NULL || pSW == NULL || rawData == NULL) { + pMeta == NULL) { uError("invalid parameter in %s", __func__); return TSDB_CODE_INVALID_PARA; } @@ -2168,7 +2173,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, uint32_t dataLen) { void* rawData = getRawDataFromRes(pRetrieve); RAW_NULL_CHECK(rawData); - uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); + uTrace(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); @@ -2251,7 +2256,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, uint32_t dataLen) void* rawData = getRawDataFromRes(pRetrieve); RAW_NULL_CHECK(rawData); - uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); + uTrace(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); @@ -2298,6 +2303,90 @@ end: return code; } +static int32_t tmqWriteRawRawDataImpl(TAOS* taos, void* data, uint32_t dataLen) { + if (taos == NULL || data == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } + int32_t code = TSDB_CODE_SUCCESS; + SQuery* pQuery = NULL; + SHashObj* pVgroupHash = NULL; + SMqRspObj rspObj = {0}; + SDecoder decoder = {0}; + + SRequestObj* pRequest = NULL; + SCatalog* pCatalog = NULL; + SRequestConnInfo conn = {0}; + + RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn)); + uDebug(LOG_ID_TAG " write raw rawdata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); + RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeMqDataRsp, &rspObj)); + + SHashObj* pVgHash = NULL; + SHashObj* pNameHash = NULL; + SHashObj* pMetaHash = NULL; + RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos)); + int retry = 0; + while (1) { + RAW_RETURN_CHECK(smlInitHandle(&pQuery)); + uDebug(LOG_ID_TAG " write raw rawdata block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum); + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(pQuery)->pRoot; + pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + RAW_NULL_CHECK(pVgroupHash); + pStmt->pVgDataBlocks = taosArrayInit(8, POINTER_BYTES); + RAW_NULL_CHECK(pStmt->pVgDataBlocks); + + while (++rspObj.resIter < rspObj.dataRsp.blockNum) { + const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter); + RAW_NULL_CHECK(tbName); + void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter); + RAW_NULL_CHECK(pRetrieve); + void* rawData = getRawDataFromRes(pRetrieve); + RAW_NULL_CHECK(rawData); + + uTrace(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); + SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; + tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); + tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); + + // find schema data info + STableMeta* pTableMeta = NULL; + RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn, &pName, + &pTableMeta, NULL, NULL, retry)); + char err[ERR_MSG_LEN] = {0}; + code = rawBlockBindRawData(pVgroupHash, pStmt->pVgDataBlocks, pTableMeta, rawData); + if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("table:%s, err:%s", pName.tname, err); + goto end; + } + } + taosHashCleanup(pVgroupHash); + pVgroupHash = NULL; + + RAW_RETURN_CHECK(smlBuildOutputRaw(pQuery, pVgHash)); + launchQueryImpl(pRequest, pQuery, true, NULL); + code = pRequest->code; + + if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) { + uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code)); + qDestroyQuery(pQuery); + pQuery = NULL; + rspObj.resIter = -1; + continue; + } + break; + } + + end: + uDebug(LOG_ID_TAG " write raw rawdata return, msg:%s", LOG_ID_VALUE, tstrerror(code)); + tDeleteMqDataRsp(&rspObj.dataRsp); + tDecoderClear(&decoder); + qDestroyQuery(pQuery); + taosHashCleanup(pVgroupHash); + destroyRequest(pRequest); + return code; +} + static void processSimpleMeta(SMqMetaRsp* pMetaRsp, cJSON** meta) { if (pMetaRsp == NULL || meta == NULL) { uError("invalid parameter in %s", __func__); @@ -2472,6 +2561,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { uError("invalid parameter in %s", __func__); return TSDB_CODE_INVALID_PARA; } + *raw = (tmq_raw_data){0}; SMqRspObj* rspObj = ((SMqRspObj*)res); if (TD_RES_TMQ_META(res)) { raw->raw = rspObj->metaRsp.metaRsp; @@ -2499,6 +2589,12 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { raw->raw_len = rspObj->batchMetaRsp.metaBuffLen; raw->raw_type = rspObj->resType; uDebug("tmq get raw batch meta:%p", raw); + } else if (TD_RES_TMQ_RAW(res)) { + raw->raw = rspObj->dataRsp.rawData; + rspObj->dataRsp.rawData = NULL; + raw->raw_len = rspObj->dataRsp.len; + raw->raw_type = rspObj->resType; + uDebug("tmq get raw raw:%p", raw); } else { uError("tmq get raw error type:%d", *(int8_t*)res); return TSDB_CODE_TMQ_INVALID_MSG; @@ -2508,8 +2604,11 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { void tmq_free_raw(tmq_raw_data raw) { uDebug("tmq free raw data type:%d", raw.raw_type); - if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) { + if (raw.raw_type == RES_TYPE__TMQ || + raw.raw_type == RES_TYPE__TMQ_METADATA) { taosMemoryFree(raw.raw); + } else if(raw.raw_type == RES_TYPE__TMQ_RAWDATA && raw.raw != NULL){ + taosMemoryFree(POINTER_SHIFT(raw.raw, - sizeof(SMqRspHead))); } (void)memset(terrMsg, 0, ERR_MSG_LEN); } @@ -2559,6 +2658,8 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) return taosDeleteData(taos, buf, len); } else if (type == RES_TYPE__TMQ_METADATA) { return tmqWriteRawMetaDataImpl(taos, buf, len); + } else if (type == RES_TYPE__TMQ_RAWDATA) { + return tmqWriteRawRawDataImpl(taos, buf, len); } else if (type == RES_TYPE__TMQ) { return tmqWriteRawDataImpl(taos, buf, len); } else if (type == RES_TYPE__TMQ_BATCH_META) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 1d461d5a49..743b94b78e 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -42,18 +42,19 @@ #define PROCESS_POLL_RSP(FUNC,DATA) \ SDecoder decoder = {0}; \ - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); \ + tDecoderInit(&decoder, POINTER_SHIFT(pRspWrapper->pollRsp.data, sizeof(SMqRspHead)), pRspWrapper->pollRsp.len - sizeof(SMqRspHead)); \ if (FUNC(&decoder, DATA) < 0) { \ tDecoderClear(&decoder); \ code = terrno; \ goto END;\ }\ tDecoderClear(&decoder);\ - (void)memcpy(DATA, pMsg->pData, sizeof(SMqRspHead)); + (void)memcpy(DATA, pRspWrapper->pollRsp.data, sizeof(SMqRspHead)); #define DELETE_POLL_RSP(FUNC,DATA) \ SMqPollRspWrapper* pRsp = &rspWrapper->pollRsp;\ - taosMemoryFreeClear(pRsp->pEpset);\ + taosMemoryFreeClear(pRsp->pEpset); \ + taosMemoryFreeClear(pRsp->data); \ FUNC(DATA); enum { @@ -92,6 +93,7 @@ struct tmq_conf_t { int8_t snapEnable; int8_t replayEnable; int8_t sourceExcluded; // do not consume, bit + int8_t rawData; // fetch raw data uint16_t port; int32_t autoCommitInterval; int32_t sessionTimeoutMs; @@ -121,6 +123,7 @@ struct tmq_t { int8_t resetOffsetCfg; int8_t replayEnable; int8_t sourceExcluded; // do not consume, bit + int8_t rawData; // fetch raw data int64_t consumerId; tmq_commit_cb* commitCb; void* commitCbUserParam; @@ -188,6 +191,8 @@ typedef struct { SMqClientTopic* topicHandle; uint64_t reqId; SEpSet* pEpset; + void* data; + uint32_t len; union { struct{ SMqRspHead head; @@ -491,11 +496,17 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } if (strcasecmp(key, "msg.consume.excluded") == 0) { - int64_t tmp; + int64_t tmp = 0; code = taosStr2int64(value, &tmp); conf->sourceExcluded = (0 == code && tmp != 0) ? TD_REQ_FROM_TAOX : 0; return TMQ_CONF_OK; } + if (strcasecmp(key, "msg.consume.rawdata") == 0) { + int64_t tmp = 0; + code = taosStr2int64(value, &tmp); + conf->rawData = (0 == code && tmp != 0) ? 1 : 0; + return TMQ_CONF_OK; + } if (strcasecmp(key, "td.connect.db") == 0) { return TMQ_CONF_OK; @@ -812,7 +823,7 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c goto end; } - if (TD_RES_TMQ(pRes) || TD_RES_TMQ_META(pRes) || + if (TD_RES_TMQ(pRes) || TD_RES_TMQ_RAW(pRes) || TD_RES_TMQ_META(pRes) || TD_RES_TMQ_METADATA(pRes) || TD_RES_TMQ_BATCH_META(pRes)) { SMqRspObj* pRspObj = (SMqRspObj*)pRes; pTopicName = pRspObj->topic; @@ -1082,6 +1093,7 @@ void tmqSendHbReq(void* param, void* tmrId) { sendInfo->fp = tmqHbCb; sendInfo->msgType = TDMT_MND_TMQ_HB; + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int32_t code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); @@ -1122,6 +1134,8 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { DELETE_POLL_RSP(tDeleteMqMetaRsp,&pRsp->metaRsp) } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { DELETE_POLL_RSP(tDeleteMqBatchMetaRsp,&pRsp->batchMetaRsp) + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { + DELETE_POLL_RSP(tDeleteMqRawDataRsp, &pRsp->dataRsp) } } @@ -1733,6 +1747,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->resetOffsetCfg = conf->resetOffset; pTmq->replayEnable = conf->replayEnable; pTmq->sourceExcluded = conf->sourceExcluded; + pTmq->rawData = conf->rawData; pTmq->enableBatchMeta = conf->enableBatchMeta; tstrncpy(pTmq->user, user, TSDB_USER_LEN); if (taosGetFqdn(pTmq->fqdn) != 0) { @@ -2057,23 +2072,14 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { goto END; } rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType; - tqDebugC("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, type %d,QID:0x%" PRIx64, tmq->consumerId, vgId, rspType, requestId); - if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { - PROCESS_POLL_RSP(tDecodeMqDataRsp, &pRspWrapper->pollRsp.dataRsp) - } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { - PROCESS_POLL_RSP(tDecodeMqMetaRsp, &pRspWrapper->pollRsp.metaRsp) - } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - PROCESS_POLL_RSP(tDecodeSTaosxRsp, &pRspWrapper->pollRsp.dataRsp) - } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { - PROCESS_POLL_RSP(tSemiDecodeMqBatchMetaRsp, &pRspWrapper->pollRsp.batchMetaRsp) - } else { // invalid rspType - tqErrorC("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); - code = TSDB_CODE_TSC_INTERNAL_ERROR; - goto END; - } + tqDebugC("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, type %d(%s),QID:0x%" PRIx64, tmq->consumerId, vgId, rspType, tmqMsgTypeStr[rspType], requestId); + pRspWrapper->tmqRspType = rspType; pRspWrapper->pollRsp.reqId = requestId; pRspWrapper->pollRsp.pEpset = pMsg->pEpSet; + pRspWrapper->pollRsp.data = pMsg->pData; + pRspWrapper->pollRsp.len = pMsg->len; + pMsg->pData = NULL; pMsg->pEpSet = NULL; END: @@ -2087,8 +2093,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosFreeQitem(pRspWrapper); tqErrorC("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); } else { - tqDebugC("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d,QID:0x%" PRIx64, - tmq ? tmq->consumerId : 0, rspType, vgId, taosQueueItemSize(tmq->mqueue), requestId); + tqDebugC("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d(%s), vgId:%d, total in queue:%d,QID:0x%" PRIx64, + tmq ? tmq->consumerId : 0, rspType, tmqMsgTypeStr[rspType], vgId, taosQueueItemSize(tmq->mqueue), requestId); } } @@ -2122,6 +2128,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->reqId = generateRequestId(); pReq->enableReplay = tmq->replayEnable; pReq->sourceExcluded = tmq->sourceExcluded; + pReq->rawData = tmq->rawData; pReq->enableBatchMeta = tmq->enableBatchMeta; } @@ -2415,6 +2422,35 @@ static int32_t processMqRspError(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ return code; } + +static int32_t processWrapperData(SMqRspWrapper* pRspWrapper){ + int32_t code = 0; + if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { + PROCESS_POLL_RSP(tDecodeMqDataRsp, &pRspWrapper->pollRsp.dataRsp) + pRspWrapper->pollRsp.dataRsp.data = pRspWrapper->pollRsp.data; + pRspWrapper->pollRsp.data = NULL; + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { + PROCESS_POLL_RSP(tDecodeMqMetaRsp, &pRspWrapper->pollRsp.metaRsp) + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { + PROCESS_POLL_RSP(tDecodeSTaosxRsp, &pRspWrapper->pollRsp.dataRsp) + pRspWrapper->pollRsp.dataRsp.data = pRspWrapper->pollRsp.data; + pRspWrapper->pollRsp.data = NULL; + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { + PROCESS_POLL_RSP(tSemiDecodeMqBatchMetaRsp, &pRspWrapper->pollRsp.batchMetaRsp) + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { + PROCESS_POLL_RSP(tDecodeMqRawDataRsp, &pRspWrapper->pollRsp.dataRsp) + pRspWrapper->pollRsp.dataRsp.len = pRspWrapper->pollRsp.len - sizeof(SMqRspHead); + pRspWrapper->pollRsp.dataRsp.rawData = POINTER_SHIFT(pRspWrapper->pollRsp.data, sizeof(SMqRspHead)); + pRspWrapper->pollRsp.data = NULL; + } else { + tqErrorC("invalid rsp msg received, type:%d ignored", pRspWrapper->tmqRspType); + code = TSDB_CODE_TSC_INTERNAL_ERROR; + goto END; + } + END: + return code; +} + static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ int32_t code = 0; SMqRspObj* pRspObj = NULL; @@ -2427,6 +2463,10 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ return pRspObj; } + code = processWrapperData(pRspWrapper); + if (code != 0) { + goto END; + } SMqPollRspWrapper* pollRspWrapper = &pRspWrapper->pollRsp; taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; @@ -2442,7 +2482,9 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ pVg->epSet = *pollRspWrapper->pEpset; } - if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP || pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { + if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP || + pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP || + pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { updateVgInfo(pVg, &pollRspWrapper->dataRsp.reqOffset, &pollRspWrapper->dataRsp.rspOffset, pollRspWrapper->head.walsver, pollRspWrapper->head.walever, tmq->consumerId, pollRspWrapper->dataRsp.blockNum != 0); @@ -2459,12 +2501,15 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ tqErrorC("consumer:0x%" PRIx64 " failed to allocate memory for meta rsp", tmq->consumerId); goto END; } - pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP ? RES_TYPE__TMQ : RES_TYPE__TMQ_METADATA; + pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP ? RES_TYPE__TMQ_RAWDATA : + (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP ? RES_TYPE__TMQ : RES_TYPE__TMQ_METADATA); int64_t numOfRows = 0; - tmqBuildRspFromWrapperInner(pollRspWrapper, pVg, &numOfRows, pRspObj); - tmq->totalRows += numOfRows; + if (pRspWrapper->tmqRspType != TMQ_MSG_TYPE__POLL_RAW_DATA_RSP){ + tmqBuildRspFromWrapperInner(pollRspWrapper, pVg, &numOfRows, pRspObj); + tmq->totalRows += numOfRows; + } pVg->emptyBlockReceiveTs = 0; - if (tmq->replayEnable) { + if (tmq->replayEnable && pRspWrapper->tmqRspType != TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { pVg->blockReceiveTs = taosGetTimestampMs(); pVg->blockSleepForReplay = pRspObj->dataRsp.sleepTime; if (pVg->blockSleepForReplay > 0) { @@ -2523,6 +2568,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq) { returnVal = processMqRsp(tmq, pRspWrapper); code = terrno; } + tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); if(returnVal != NULL || code != 0){ @@ -2575,7 +2621,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { END: terrno = code; - if (tmq != NULL) { + if (tmq != NULL && terrno != 0) { tqErrorC("consumer:0x%" PRIx64 " poll error at line:%d, msg:%s", tmq->consumerId, lino, tstrerror(terrno)); } return NULL; @@ -2688,6 +2734,8 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) { return TMQ_RES_METADATA; } else if (TD_RES_TMQ_BATCH_META(res)) { return TMQ_RES_TABLE_META; + } else if (TD_RES_TMQ_RAW(res)) { + return TMQ_RES_RAWDATA; } else { return TMQ_RES_INVALID; } @@ -2697,7 +2745,7 @@ const char* tmq_get_topic_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { char* tmp = strchr(((SMqRspObj*)res)->topic, '.'); if (tmp == NULL) { @@ -2714,7 +2762,7 @@ const char* tmq_get_db_name(TAOS_RES* res) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) { char* tmp = strchr(((SMqRspObj*)res)->db, '.'); if (tmp == NULL) { @@ -2730,7 +2778,7 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (res == NULL) { return TSDB_CODE_INVALID_PARA; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) { return ((SMqRspObj*)res)->vgId; } else { @@ -2742,7 +2790,7 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { if (res == NULL) { return TSDB_CODE_INVALID_PARA; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; STqOffsetVal* pOffset = &pRspObj->dataRsp.reqOffset; if (pOffset->type == TMQ_OFFSET__LOG) { @@ -2767,7 +2815,7 @@ const char* tmq_get_table_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) ) { SMqRspObj* pRspObj = (SMqRspObj*)res; SMqDataRsp* data = &pRspObj->dataRsp; if (!data->withTbName || data->blockTbName == NULL || pRspObj->resIter < 0 || @@ -3670,4 +3718,4 @@ TAOS* tmq_get_connect(tmq_t* tmq) { return (TAOS*)(&(tmq->pTscObj->id)); } return NULL; -} \ No newline at end of file +} diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 052fcbe2ba..3e24265876 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -9219,6 +9219,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->enableReplay)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->sourceExcluded)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->enableBatchMeta)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->rawData)); tEndEncode(&encoder); @@ -9272,6 +9273,10 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { pReq->enableBatchMeta = false; } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->rawData)); + } + tEndDecode(&decoder); _exit: @@ -9279,7 +9284,13 @@ _exit: return code; } -void tDestroySMqPollReq(SMqPollReq *pReq) { tOffsetDestroy(&pReq->reqOffset); } +void tDestroySMqPollReq(SMqPollReq *pReq) { + tOffsetDestroy(&pReq->reqOffset); + if (pReq->uidHash != NULL) { + taosHashCleanup(pReq->uidHash); + pReq->uidHash = NULL; + } +} int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { int32_t code = 0; int32_t lino; @@ -11448,12 +11459,14 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { } for (int32_t i = 0; i < pRsp->blockNum; i++) { - void *data; - uint64_t bLen; - TAOS_CHECK_EXIT(tDecodeBinaryAlloc(pDecoder, &data, &bLen)); + void *data = NULL; + uint32_t bLen = 0; + TAOS_CHECK_EXIT(tDecodeBinary(pDecoder, (uint8_t**)&data, &bLen)); if (taosArrayPush(pRsp->blockData, &data) == NULL) { TAOS_CHECK_EXIT(terrno); } + pRsp->blockDataElementFree = false; + int32_t len = bLen; if (taosArrayPush(pRsp->blockDataLen, &len) == NULL) { TAOS_CHECK_EXIT(terrno); @@ -11499,10 +11512,25 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { return 0; } +int32_t tDecodeMqRawDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset)); + TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->blockNum)); +_exit: + return code; +} + static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) { taosArrayDestroy(pRsp->blockDataLen); pRsp->blockDataLen = NULL; - taosArrayDestroyP(pRsp->blockData, NULL); + if (pRsp->blockDataElementFree){ + taosArrayDestroyP(pRsp->blockData, NULL); + } else { + taosArrayDestroy(pRsp->blockData); + } pRsp->blockData = NULL; taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); pRsp->blockSchema = NULL; @@ -11510,6 +11538,7 @@ static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) { pRsp->blockTbName = NULL; tOffsetDestroy(&pRsp->reqOffset); tOffsetDestroy(&pRsp->rspOffset); + taosMemoryFreeClear(pRsp->data); } void tDeleteMqDataRsp(SMqDataRsp *rsp) { tDeleteMqDataRspCommon(rsp); } @@ -11572,6 +11601,14 @@ void tDeleteSTaosxRsp(SMqDataRsp *pRsp) { pRsp->createTableReq = NULL; } +void tDeleteMqRawDataRsp(SMqDataRsp *pRsp) { + tOffsetDestroy(&pRsp->reqOffset); + tOffsetDestroy(&pRsp->rspOffset); + if (pRsp->rawData != NULL){ + taosMemoryFree(POINTER_SHIFT(pRsp->rawData, - sizeof(SMqRspHead))); + } +} + int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pReq->tbname)); TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pReq->startTs)); @@ -11658,6 +11695,26 @@ int32_t tDecodeSBatchDeleteReqSetCtime(SDecoder *pDecoder, SBatchDeleteReq *pReq _exit: return code; } +int32_t transformRawSSubmitTbData(void* data, int64_t suid, int64_t uid, int32_t sver){ + int32_t code = 0; + int32_t lino = 0; + SDecoder decoder = {0}; + tDecoderInit(&decoder, (uint8_t *)POINTER_SHIFT(data, INT_BYTES), *(uint32_t*)data); + + int32_t flags = 0; + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &flags)); + flags |= TD_REQ_FROM_TAOX; + flags &= ~SUBMIT_REQ_AUTO_CREATE_TABLE; + + SEncoder encoder = {0}; + tEncoderInit(&encoder, (uint8_t *)POINTER_SHIFT(data, INT_BYTES), *(uint32_t*)data); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, flags)); + TAOS_CHECK_EXIT(tEncodeI64(&encoder, suid)); + TAOS_CHECK_EXIT(tEncodeI64(&encoder, uid)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, sver)); + _exit: + return code; +} static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) { int32_t code = 0; @@ -11705,14 +11762,20 @@ _exit: return code; } -static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData) { +static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData, void* rawData) { int32_t code = 0; int32_t lino; int32_t flags; uint8_t version; + uint8_t* dataAfterCreate = NULL; + uint8_t* dataStart = pCoder->data + pCoder->pos; + uint32_t posAfterCreate = 0; + TAOS_CHECK_EXIT(tStartDecode(pCoder)); + uint32_t pos = pCoder->pos; TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &flags)); + uint32_t flagsLen = pCoder->pos - pos; pSubmitTbData->flags = flags & 0xff; version = (flags >> 8) & 0xff; @@ -11724,6 +11787,8 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa } TAOS_CHECK_EXIT(tDecodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq)); + dataAfterCreate = pCoder->data + pCoder->pos; + posAfterCreate = pCoder->pos; } // submit data @@ -11732,7 +11797,7 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &pSubmitTbData->sver)); if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { - uint64_t nColData; + uint64_t nColData = 0; TAOS_CHECK_EXIT(tDecodeU64v(pCoder, &nColData)); @@ -11745,7 +11810,7 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa TAOS_CHECK_EXIT(tDecodeColData(version, pCoder, taosArrayReserve(pSubmitTbData->aCol, 1))); } } else { - uint64_t nRow; + uint64_t nRow = 0; TAOS_CHECK_EXIT(tDecodeU64v(pCoder, &nRow)); pSubmitTbData->aRowP = taosArrayInit(nRow, sizeof(SRow *)); @@ -11768,6 +11833,15 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa TAOS_CHECK_EXIT(tDecodeI64(pCoder, &pSubmitTbData->ctimeMs)); } + if (rawData != NULL){ + if (dataAfterCreate != NULL){ + TAOS_MEMCPY(dataAfterCreate - INT_BYTES - flagsLen, dataStart, INT_BYTES + flagsLen); + *(int32_t*)(dataAfterCreate - INT_BYTES - flagsLen) = pCoder->pos - posAfterCreate + flagsLen; + *(void**)rawData = dataAfterCreate - INT_BYTES - flagsLen; + }else{ + *(void**)rawData = dataStart; + } + } tEndDecode(pCoder); _exit: @@ -11780,15 +11854,27 @@ int32_t tEncodeSubmitReq(SEncoder *pCoder, const SSubmitReq2 *pReq) { TAOS_CHECK_EXIT(tStartEncode(pCoder)); TAOS_CHECK_EXIT(tEncodeU64v(pCoder, taosArrayGetSize(pReq->aSubmitTbData))); - for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { - TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i))); + if (pReq->raw){ + for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { + void* data = taosArrayGetP(pReq->aSubmitTbData, i); + if (pCoder->data != NULL){ + TAOS_MEMCPY(pCoder->data + pCoder->pos, data, *(uint32_t*)data + INT_BYTES); + + } + pCoder->pos += *(uint32_t*)data + INT_BYTES; + } + } else{ + for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { + TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i))); + } } + tEndEncode(pCoder); _exit: return code; } -int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) { +int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq, SArray* rawList) { int32_t code = 0; memset(pReq, 0, sizeof(*pReq)); @@ -11812,7 +11898,8 @@ int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) { } for (uint64_t i = 0; i < nSubmitTbData; i++) { - if (tDecodeSSubmitTbData(pCoder, taosArrayReserve(pReq->aSubmitTbData, 1)) < 0) { + SSubmitTbData* data = taosArrayReserve(pReq->aSubmitTbData, 1); + if (tDecodeSSubmitTbData(pCoder, data, rawList != NULL ? taosArrayReserve(rawList, 1) : NULL) < 0) { code = TSDB_CODE_INVALID_MSG; goto _exit; } @@ -11821,13 +11908,6 @@ int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) { tEndDecode(pCoder); _exit: - if (code) { - if (pReq->aSubmitTbData) { - // todo - taosArrayDestroy(pReq->aSubmitTbData); - pReq->aSubmitTbData = NULL; - } - } return code; } @@ -11883,12 +11963,15 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) { void tDestroySubmitReq(SSubmitReq2 *pReq, int32_t flag) { if (pReq->aSubmitTbData == NULL) return; - int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); - SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); + if (!pReq->raw){ + int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); + SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); - for (int32_t i = 0; i < nSubmitTbData; i++) { - tDestroySubmitTbData(&aSubmitTbData[i], flag); + for (int32_t i = 0; i < nSubmitTbData; i++) { + tDestroySubmitTbData(&aSubmitTbData[i], flag); + } } + taosArrayDestroy(pReq->aSubmitTbData); pReq->aSubmitTbData = NULL; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index cb305cb6c7..4a9a5ac274 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2731,7 +2731,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat terrno = 0; if (NULL == pReq) { - if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { code = terrno; goto _end; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 50d75c4838..f0e7af50f3 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -242,11 +242,11 @@ SSDataBlock *tqGetResultBlock(STqReader *pReader); int64_t tqGetResultBlockTime(STqReader *pReader); int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, const char *id); -int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver, SArray* rawList); +void tqReaderClearSubmitMsg(STqReader *pReader); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr); -int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, - int64_t *createTime); +int32_t tqRetrieveTaosxBlock(STqReader *pReader, SMqDataRsp* pRsp, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, SArray* rawList, int8_t fetchMeta); int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished); // sma diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e0bf51b333..50d6eb0090 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -90,6 +90,7 @@ typedef struct { // for replay SSDataBlock* block; int64_t blockTime; + SHashObj* tableCreateTimeHash; // for process create table msg in submit if fetch raw data } STqHandle; struct STQ { @@ -117,8 +118,8 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded); -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, const SMqPollReq* pRequest); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, SMqDataRsp* pRsp, int32_t type, int32_t vgId); void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); @@ -147,7 +148,7 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name); // tq util int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type, EStreamType blockType); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset); void tqUpdateNodeStage(STQ* pTq, bool isLeader); @@ -183,6 +184,7 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode* #define TQ_SUBSCRIBE_NAME "subscribe" #define TQ_OFFSET_NAME "offset-ver0" #define TQ_POLL_MAX_TIME 1000 +#define TQ_POLL_MAX_BYTES 1048576 #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 5a61c1c124..402ab8da7e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -167,7 +167,8 @@ int32_t metaDropMultipleTables(SMeta* pMeta, int64_t version, SArray* tb int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs); -SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t* createTime); +SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); +int64_t metaGetTableCreateTime(SMeta *pMeta, tb_uid_t uid, int lock); int32_t metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema); int32_t metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index b367232e2d..169adf219f 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -378,7 +378,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) { return 0; } -SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime) { +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { void *pData = NULL; int nData = 0; int64_t version; @@ -414,9 +414,6 @@ _query: } } else if (me.type == TSDB_CHILD_TABLE) { uid = me.ctbEntry.suid; - if (createTime != NULL){ - *createTime = me.ctbEntry.btime; - } tDecoderClear(&dc); goto _query; } else { @@ -455,6 +452,46 @@ _err: return NULL; } +int64_t metaGetTableCreateTime(SMeta *pMeta, tb_uid_t uid, int lock) { + void *pData = NULL; + int nData = 0; + int64_t version = 0; + SDecoder dc = {0}; + int64_t createTime = INT64_MAX; + if (lock) { + metaRLock(pMeta); + } + + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData) < 0) { + goto _exit; + } + + version = ((SUidIdxVal *)pData)[0].version; + + if (tdbTbGet(pMeta->pTbDb, &(STbDbKey){.uid = uid, .version = version}, sizeof(STbDbKey), &pData, &nData) != 0) { + goto _exit; + } + + SMetaEntry me = {0}; + tDecoderInit(&dc, pData, nData); + int32_t code = metaDecodeEntry(&dc, &me); + if (code) { + tDecoderClear(&dc); + goto _exit; + } + if (me.type == TSDB_CHILD_TABLE) { + createTime = me.ctbEntry.btime; + } + tDecoderClear(&dc); + + _exit: + if (lock) { + metaULock(pMeta); + } + tdbFree(pData); + return createTime; +} + SMCtbCursor *metaOpenCtbCursor(void *pVnode, tb_uid_t uid, int lock) { SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMCtbCursor *pCtbCur = NULL; @@ -627,7 +664,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { STSchema *pTSchema = NULL; SSchemaWrapper *pSW = NULL; - pSW = metaGetTableSchema(pMeta, uid, sver, lock, NULL); + pSW = metaGetTableSchema(pMeta, uid, sver, lock); if (!pSW) return NULL; pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 7374b9ceb5..b227653e5e 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -548,7 +548,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) { void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) { bool ret = false; - SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1, NULL); + SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { ret = true; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1c0f73249c..66e64b91d9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -57,6 +57,7 @@ void tqDestroyTqHandle(void* data) { if (pData->pRef) { walCloseRef(pData->pRef->pWal, pData->pRef->refId); } + taosHashCleanup(pData->tableCreateTimeHash); } static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) { @@ -211,7 +212,7 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { tDeleteMqDataRsp(&dataRsp); } -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type, +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, SMqDataRsp* pRsp, int32_t type, int32_t vgId) { if (pHandle == NULL || pMsg == NULL || pReq == NULL || pRsp == NULL) { return TSDB_CODE_INVALID_PARA; @@ -414,7 +415,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { terrno = TSDB_CODE_INVALID_MSG; goto END; } - + if (req.rawData == 1){ + req.uidHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (req.uidHash == NULL) { + tqError("tq poll rawData taosHashInit failed"); + code = terrno; + goto END; + } + } int64_t consumerId = req.consumerId; int32_t reqEpoch = req.epoch; STqOffsetVal reqOffset = req.reqOffset; diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 580828b089..b84dcc4703 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -343,6 +343,7 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle) { tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL); taosArrayDestroy(tbUidList); } + handle->tableCreateTimeHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); END: return code; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 937ca80bcc..d650ae9751 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -283,7 +283,7 @@ void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) { return; } bool ret = false; - SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1, NULL); + SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1); if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { ret = true; } @@ -352,7 +352,7 @@ int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) { return TSDB_CODE_INVALID_PARA; } if (walReaderSeekVer(pReader->pWalReader, ver) < 0) { - return -1; + return terrno; } tqDebug("wal reader seek to ver:%" PRId64 " %s", ver, id); return 0; @@ -485,14 +485,14 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); int64_t ver = pWalReader->pHead->head.version; - if (tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver) != 0) { + if (tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver, NULL) != 0) { return false; } pReader->nextBlk = 0; } } -int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { +int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver, SArray* rawList) { if (pReader == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -500,11 +500,11 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i pReader->msg.msgLen = msgLen; pReader->msg.ver = ver; - tqDebug("tq reader set msg pointer:%p, msg len:%d", msgStr, msgLen); + tqTrace("tq reader set msg pointer:%p, msg len:%d", msgStr, msgLen); SDecoder decoder = {0}; tDecoderInit(&decoder, pReader->msg.msgStr, pReader->msg.msgLen); - int32_t code = tDecodeSubmitReq(&decoder, &pReader->submit); + int32_t code = tDecodeSubmitReq(&decoder, &pReader->submit, rawList); tDecoderClear(&decoder); if (code != 0) { @@ -514,6 +514,13 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i return code; } +void tqReaderClearSubmitMsg(STqReader *pReader) { + tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->nextBlk = 0; + pReader->msg.msgStr = NULL; +} + + SWalReader* tqGetWalReader(STqReader* pReader) { if (pReader == NULL) { return NULL; @@ -551,17 +558,15 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); TSDB_CHECK_CONDITION(ret == NULL, code, lino, END, true); - tqDebug("iterator data block in hash continue, progress:%d/%d, total queried tables:%d, uid:%"PRId64, pReader->nextBlk, blockSz, taosHashGetSize(pReader->tbIdHash), uid); + tqTrace("iterator data block in hash jump block, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid); pReader->nextBlk++; } - tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->nextBlk = 0; - pReader->msg.msgStr = NULL; - tqDebug("iterator data block end, block progress:%d/%d, uid:%"PRId64, pReader->nextBlk, blockSz, uid); + tqReaderClearSubmitMsg(pReader); + tqTrace("iterator data block end, total block num:%d, uid:%"PRId64, blockSz, uid); END: - tqDebug("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); + tqTrace("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); return code; } @@ -581,17 +586,14 @@ bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) { uid = pSubmitTbData->uid; void* ret = taosHashGet(filterOutUids, &pSubmitTbData->uid, sizeof(int64_t)); TSDB_CHECK_NULL(ret, code, lino, END, true); - tqDebug("iterator data block in hash continue, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid); + tqTrace("iterator data block in hash jump block, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid); pReader->nextBlk++; } - - tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->nextBlk = 0; - pReader->msg.msgStr = NULL; - tqDebug("iterator data block end, block progress:%d/%d, uid:%"PRId64, pReader->nextBlk, blockSz, uid); + tqReaderClearSubmitMsg(pReader); + tqTrace("iterator data block end, total block num:%d, uid:%"PRId64, blockSz, uid); END: - tqDebug("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); + tqTrace("%s:%d get data:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); return code; } @@ -740,7 +742,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* (pReader->cachedSchemaVer != sversion)) { tDeleteSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, NULL); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64 "version %d, possibly dropped table", @@ -934,7 +936,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData TQ_NULL_GO_TO_END(pCol); int32_t numOfRows = pCol->nVal; int32_t numOfCols = taosArrayGetSize(pCols); - tqDebug("vgId:%d, tqProcessColData start, col num: %d, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows); + tqTrace("vgId:%d, tqProcessColData start, col num: %d, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows); for (int32_t i = 0; i < numOfRows; i++) { bool buildNew = false; @@ -974,7 +976,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData } SSDataBlock* pLastBlock = taosArrayGetLast(blocks); pLastBlock->info.rows = curRow - lastRow; - tqDebug("vgId:%d, tqProcessColData end, col num: %d, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows, (int)taosArrayGetSize(blocks)); + tqTrace("vgId:%d, tqProcessColData end, col num: %d, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows, (int)taosArrayGetSize(blocks)); END: if (code != TSDB_CODE_SUCCESS) { tqError("vgId:%d, process col data failed, code:%d", pReader->pWalReader->pWal->cfg.vgId, code); @@ -997,7 +999,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra int32_t numOfRows = taosArrayGetSize(pRows); pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); TQ_NULL_GO_TO_END(pTSchema); - tqDebug("vgId:%d, tqProcessRowData start, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows); + tqTrace("vgId:%d, tqProcessRowData start, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows); for (int32_t i = 0; i < numOfRows; i++) { bool buildNew = false; @@ -1036,7 +1038,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra SSDataBlock* pLastBlock = taosArrayGetLast(blocks); pLastBlock->info.rows = curRow - lastRow; - tqDebug("vgId:%d, tqProcessRowData end, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows, (int)taosArrayGetSize(blocks)); + tqTrace("vgId:%d, tqProcessRowData end, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows, (int)taosArrayGetSize(blocks)); END: if (code != TSDB_CODE_SUCCESS) { tqError("vgId:%d, process row data failed, code:%d", pReader->pWalReader->pWal->cfg.vgId, code); @@ -1046,8 +1048,46 @@ END: return code; } -int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, int64_t *createTime) { - tqDebug("tq reader retrieve data block msg pointer:%p, index:%d", pReader->msg.msgStr, pReader->nextBlk); +static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){ + int32_t code = 0; + int32_t lino = 0; + void* createReq = NULL; + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pCreateTbReq, code, lino, END, TSDB_CODE_INVALID_PARA); + + if (pRsp->createTableNum == 0) { + pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); + TSDB_CHECK_NULL(pRsp->createTableLen, code, lino, END, terrno); + pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); + TSDB_CHECK_NULL(pRsp->createTableReq, code, lino, END, terrno); + } + + uint32_t len = 0; + tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code); + TSDB_CHECK_CODE(code, lino, END); + createReq = taosMemoryCalloc(1, len); + TSDB_CHECK_NULL(createReq, code, lino, END, terrno); + + SEncoder encoder = {0}; + tEncoderInit(&encoder, createReq, len); + code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq); + tEncoderClear(&encoder); + TSDB_CHECK_CODE(code, lino, END); + TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableLen, &len), code, lino, END, terrno); + TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableReq, &createReq), code, lino, END, terrno); + pRsp->createTableNum++; + tqTrace("build create table info msg success"); + + END: + if (code != 0){ + tqError("%s failed at %d, failed to build create table info msg:%s", __FUNCTION__, lino, tstrerror(code)); + taosMemoryFree(createReq); + } + return code; +} + +int32_t tqRetrieveTaosxBlock(STqReader* pReader, SMqDataRsp* pRsp, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, SArray* rawList, int8_t fetchMeta) { + tqTrace("tq reader retrieve data block msg pointer:%p, index:%d", pReader->msg.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pSubmitTbData == NULL) { return terrno; @@ -1063,7 +1103,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas pReader->lastBlkUid = uid; tDeleteSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, createTime); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); @@ -1071,6 +1111,19 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas return TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; } + if (pSubmitTbData->pCreateTbReq != NULL) { + int32_t code = buildCreateTbInfo(pRsp, pSubmitTbData->pCreateTbReq); + if (code != 0) { + return code; + } + } else if (rawList != NULL) { + if (taosArrayPush(schemas, &pReader->pSchemaWrapper) == NULL){ + return terrno; + } + pReader->pSchemaWrapper = NULL; + return 0; + } + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { return tqProcessColData(pReader, pSubmitTbData, blocks, schemas); } else { diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index a65b118aea..b109be5626 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -14,6 +14,33 @@ */ #include "tq.h" +static int32_t tqAddRawDataToRsp(const void* rawData, SMqDataRsp* pRsp, int8_t precision) { + int32_t code = TDB_CODE_SUCCESS; + int32_t lino = 0; + void* buf = NULL; + + int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + *(uint32_t *)rawData + INT_BYTES; + buf = taosMemoryCalloc(1, dataStrLen); + TSDB_CHECK_NULL(buf, code, lino, END, terrno); + + SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)buf; + pRetrieve->version = RETRIEVE_TABLE_RSP_TMQ_RAW_VERSION; + pRetrieve->precision = precision; + pRetrieve->compressed = 0; + + memcpy(pRetrieve->data, rawData, *(uint32_t *)rawData + INT_BYTES); + TSDB_CHECK_NULL(taosArrayPush(pRsp->blockDataLen, &dataStrLen), code, lino, END, terrno); + TSDB_CHECK_NULL(taosArrayPush(pRsp->blockData, &buf), code, lino, END, terrno); + pRsp->blockDataElementFree = true; + + tqTrace("tqAddRawDataToRsp add block data to block array, blockDataLen:%d, blockData:%p", dataStrLen, buf); + END: + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + tqError("%s failed at %d, failed to add block data to response:%s", __FUNCTION__, lino, tstrerror(code)); + } + return code; +} static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { int32_t code = 0; @@ -25,7 +52,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, TSDB_CHECK_NULL(buf, code, lino, END, terrno); SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)buf; - pRetrieve->version = 1; + pRetrieve->version = RETRIEVE_TABLE_RSP_TMQ_VERSION; pRetrieve->precision = precision; pRetrieve->compressed = 0; pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); @@ -36,13 +63,14 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, actualLen += sizeof(SRetrieveTableRspForTmq); TSDB_CHECK_NULL(taosArrayPush(pRsp->blockDataLen, &actualLen), code, lino, END, terrno); TSDB_CHECK_NULL(taosArrayPush(pRsp->blockData, &buf), code, lino, END, terrno); + pRsp->blockDataElementFree = true; + tqTrace("tqAddBlockDataToRsp add block data to block array, blockDataLen:%d, blockData:%p", dataStrLen, buf); - buf = NULL; END: - if (code != 0){ + if (code != TSDB_CODE_SUCCESS){ + taosMemoryFree(buf); tqError("%s failed at line %d with msg:%s", __func__, lino, tstrerror(code)); } - taosMemoryFree(buf); return code; } @@ -66,7 +94,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i tqError("failed to push tbName to blockTbName:%s, uid:%"PRId64, tbName, uid); continue; } - tqDebug("add tbName to response success tbname:%s, uid:%"PRId64, tbName, uid); + tqTrace("add tbName to response success tbname:%s, uid:%"PRId64, tbName, uid); } END: @@ -234,7 +262,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat tbName = NULL; } if (pRsp->withSchema) { - SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); + pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); TSDB_CHECK_NULL(pSW, code, lino, END, terrno); TSDB_CHECK_NULL(taosArrayPush(pRsp->blockSchema, &pSW), code, lino, END, terrno); pSW = NULL; @@ -290,45 +318,8 @@ END: return code; } -static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){ - int32_t code = 0; - int32_t lino = 0; - void* createReq = NULL; - TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); - TSDB_CHECK_NULL(pCreateTbReq, code, lino, END, TSDB_CODE_INVALID_PARA); - - if (pRsp->createTableNum == 0) { - pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); - TSDB_CHECK_NULL(pRsp->createTableLen, code, lino, END, terrno); - pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); - TSDB_CHECK_NULL(pRsp->createTableReq, code, lino, END, terrno); - } - - uint32_t len = 0; - tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code); - TSDB_CHECK_CODE(code, lino, END); - createReq = taosMemoryCalloc(1, len); - TSDB_CHECK_NULL(createReq, code, lino, END, terrno); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, createReq, len); - code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq); - tEncoderClear(&encoder); - TSDB_CHECK_CODE(code, lino, END); - TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableLen, &len), code, lino, END, terrno); - TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableReq, &createReq), code, lino, END, terrno); - pRsp->createTableNum++; - tqDebug("build create table info msg success"); - -END: - if (code != 0){ - tqError("%s failed at %d, failed to build create table info msg:%s", __FUNCTION__, lino, tstrerror(code)); - taosMemoryFree(createReq); - } - return code; -} - -static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){ +static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, + const SMqPollReq* pRequest, SArray* rawList){ int32_t code = 0; int32_t lino = 0; SArray* pBlocks = NULL; @@ -342,78 +333,161 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int pSchemas = taosArrayInit(0, sizeof(void*)); TSDB_CHECK_NULL(pSchemas, code, lino, END, terrno); - SSubmitTbData* pSubmitTbDataRet = NULL; - int64_t createTime = INT64_MAX; - code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet, &createTime); + SSubmitTbData* pSubmitTbData = NULL; + code = tqRetrieveTaosxBlock(pReader, pRsp, pBlocks, pSchemas, &pSubmitTbData, rawList, pHandle->fetchMeta); TSDB_CHECK_CODE(code, lino, END); - bool tmp = (pSubmitTbDataRet->flags & sourceExcluded) != 0; + bool tmp = (pSubmitTbData->flags & pRequest->sourceExcluded) != 0; TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS); + + + int32_t blockNum = taosArrayGetSize(pBlocks) == 0 ? 1 : taosArrayGetSize(pBlocks); if (pRsp->withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; - code = tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)); + code = tqAddTbNameToRsp(pTq, uid, pRsp, blockNum); TSDB_CHECK_CODE(code, lino, END); } - if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) { - if (pSubmitTbDataRet->ctimeMs - createTime <= 1000) { // judge if table is already created to avoid sending crateTbReq - code = buildCreateTbInfo(pRsp, pSubmitTbDataRet->pCreateTbReq); - TSDB_CHECK_CODE(code, lino, END); - } - } - tmp = (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL); + + tmp = (pHandle->fetchMeta == ONLY_META && pSubmitTbData->pCreateTbReq == NULL); TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS); - for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { - SSDataBlock* pBlock = taosArrayGet(pBlocks, i); - if (pBlock == NULL) { - continue; + for (int32_t i = 0; i < blockNum; i++) { + if (taosArrayGetSize(pBlocks) == 0){ + void* rawData = taosArrayGetP(rawList, pReader->nextBlk - 1); + if (rawData == NULL) { + continue; + } + if (tqAddRawDataToRsp(rawData, pRsp, pTq->pVnode->config.tsdbCfg.precision) != 0){ + tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); + continue; + } + *totalRows += *(uint32_t *)rawData + INT_BYTES; // bytes actually + } else { + SSDataBlock* pBlock = taosArrayGet(pBlocks, i); + if (pBlock == NULL) { + continue; + } + + if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0){ + tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); + continue; + } + *totalRows += pBlock->info.rows; } - if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0){ - tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); - continue; - } - *totalRows += pBlock->info.rows; - blockDataFreeRes(pBlock); - SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); - if (taosArrayPush(pRsp->blockSchema, &pSW) == NULL){ + + void** pSW = taosArrayGet(pSchemas, i); + if (taosArrayPush(pRsp->blockSchema, pSW) == NULL){ tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); continue; } + *pSW = NULL; pRsp->blockNum++; } - tqDebug("vgId:%d, process sub data success, response blocknum:%d, rows:%d", pTq->pVnode->config.vgId, pRsp->blockNum, *totalRows); + tqTrace("vgId:%d, process sub data success, response blocknum:%d, rows:%d", pTq->pVnode->config.vgId, pRsp->blockNum, *totalRows); END: - if (code != 0){ + if (code != 0) { tqError("%s failed at %d, failed to process sub data:%s", __FUNCTION__, lino, tstrerror(code)); - taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); - taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); - } else { - taosArrayDestroy(pBlocks); - taosArrayDestroy(pSchemas); + } + taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); + taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); +} + +static void preProcessSubmitMsg(STqHandle* pHandle, const SMqPollReq* pRequest, SArray** rawList){ + STqExecHandle* pExec = &pHandle->execHandle; + STqReader* pReader = pExec->pTqReader; + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + for (int32_t i = 0; i < blockSz; i++){ + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, i); + if (pSubmitTbData== NULL){ + taosArrayDestroy(*rawList); + *rawList = NULL; + return; + } + + int64_t uid = pSubmitTbData->uid; + if (taosHashGet(pRequest->uidHash, &uid, LONG_BYTES) != NULL) { + tqDebug("poll rawdata split,uid:%" PRId64 " is already exists", uid); + terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT; + return; + } else { + int32_t code = taosHashPut(pRequest->uidHash, &uid, LONG_BYTES, &uid, LONG_BYTES); + if (code != 0){ + tqError("failed to add table uid to hash, code:%d, uid:%"PRId64, code, uid); + } + } + + if (pSubmitTbData->pCreateTbReq == NULL){ + continue; + } + + int64_t createTime = INT64_MAX; + int64_t *cTime = (int64_t*)taosHashGet(pHandle->tableCreateTimeHash, &uid, LONG_BYTES); + if (cTime != NULL){ + createTime = *cTime; + } else{ + createTime = metaGetTableCreateTime(pReader->pVnodeMeta, uid, 1); + if (createTime != INT64_MAX){ + int32_t code = taosHashPut(pHandle->tableCreateTimeHash, &uid, LONG_BYTES, &createTime, LONG_BYTES); + if (code != 0){ + tqError("failed to add table create time to hash,code:%d, uid:%"PRId64, code, uid); + } + } + } + if (pHandle->fetchMeta == WITH_DATA || pSubmitTbData->ctimeMs > createTime){ + tDestroySVCreateTbReq(pSubmitTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE); + pSubmitTbData->pCreateTbReq = NULL; + } else{ + taosArrayDestroy(*rawList); + *rawList = NULL; + } } } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) { +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, const SMqPollReq* pRequest) { int32_t code = 0; int32_t lino = 0; - TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); - TSDB_CHECK_NULL(pTq, code, lino, END, TSDB_CODE_INVALID_PARA); - TSDB_CHECK_NULL(pHandle, code, lino, END, TSDB_CODE_INVALID_PARA); - TSDB_CHECK_NULL(totalRows, code, lino, END, TSDB_CODE_INVALID_PARA); STqExecHandle* pExec = &pHandle->execHandle; STqReader* pReader = pExec->pTqReader; - code = tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); + SArray *rawList = NULL; + if (pRequest->rawData){ + rawList = taosArrayInit(0, POINTER_BYTES); + TSDB_CHECK_NULL(rawList, code, lino, END, terrno); + } + code = tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver, rawList); TSDB_CHECK_CODE(code, lino, END); + if (pRequest->rawData) { + preProcessSubmitMsg(pHandle, pRequest, &rawList); + } + // data could not contains same uid data in rawdata mode + if (pRequest->rawData != 0 && terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){ + goto END; + } + + // this submit data is metadata and previous data is rawdata + if (pRequest->rawData != 0 && *totalRows > 0 && pRsp->createTableNum == 0 && rawList == NULL){ + tqDebug("poll rawdata split,vgId:%d, this wal submit data contains metadata and previous data is data", pTq->pVnode->config.vgId); + terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT; + goto END; + } + + // this submit data is rawdata and previous data is metadata + if (pRequest->rawData != 0 && pRsp->createTableNum > 0 && rawList != NULL){ + tqDebug("poll rawdata split,vgId:%d, this wal submit data is data and previous data is metadata", pTq->pVnode->config.vgId); + terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT; + goto END; + } if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { while (tqNextBlockImpl(pReader, NULL)) { - tqProcessSubData(pTq, pHandle, pRsp, totalRows, sourceExcluded); + tqProcessSubData(pTq, pHandle, pRsp, totalRows, pRequest, rawList); } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { - tqProcessSubData(pTq, pHandle, pRsp, totalRows, sourceExcluded); + tqProcessSubData(pTq, pHandle, pRsp, totalRows, pRequest, rawList); } } END: + tqReaderClearSubmitMsg(pReader); + taosArrayDestroy(rawList); if (code != 0){ tqError("%s failed at %d, failed to scan log:%s", __FUNCTION__, lino, tstrerror(code)); } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 197a45cdb9..44d9ffda83 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -131,7 +131,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, pHandle->subKey, vgId, dataRsp.rspOffset.version); - code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, (pRequest->rawData == 1) ? TMQ_MSG_TYPE__POLL_RAW_DATA_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); tDeleteMqDataRsp(&dataRsp); *pBlockReturned = true; @@ -222,6 +222,10 @@ end: static void tDeleteCommon(void* parm) {} +#define POLL_RSP_TYPE(pRequest,taosxRsp) \ +taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : \ +(pRequest->rawData == 1 ? TMQ_MSG_TYPE__POLL_RAW_DATA_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP) + static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { int32_t vgId = TD_VID(pTq->pVnode); @@ -274,7 +278,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, - taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + POLL_RSP_TYPE(pRequest, taosxRsp), vgId); goto END; } @@ -287,7 +291,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (totalRows > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, - taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + POLL_RSP_TYPE(pRequest, taosxRsp), vgId); goto END; } @@ -370,12 +374,19 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, .ver = pHead->version, }; - TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest->sourceExcluded)); + TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest)); - if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > TMIN(TQ_POLL_MAX_TIME, pRequest->timeout))) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); + if ((pRequest->rawData == 0 && totalRows >= tmqRowSize) || + (taosGetTimestampMs() - st > TMIN(TQ_POLL_MAX_TIME, pRequest->timeout)) || + (pRequest->rawData != 0 && (totalRows >= TQ_POLL_MAX_BYTES || + taosArrayGetSize(taosxRsp.blockData) > tmqRowSize || + terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT))) { +// tqDebug("start to send rsp, block num:%d, totalRows:%d, createTableNum:%d, terrno:%d", +// (int)taosArrayGetSize(taosxRsp.blockData), totalRows, taosxRsp.createTableNum, terrno); + tqOffsetResetToLog(&taosxRsp.rspOffset, terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT ? fetchVer : fetchVer + 1); code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, - taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + POLL_RSP_TYPE(pRequest, taosxRsp), vgId); + if (terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){terrno = 0;} goto END; } else { fetchVer++; @@ -526,7 +537,7 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll return 0; } -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever) { if (pRpcHandleInfo == NULL || pRsp == NULL) { return TSDB_CODE_TMQ_INVALID_MSG; @@ -534,7 +545,12 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* int32_t len = 0; int32_t code = 0; - if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + if (type == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP){ + pRsp->withSchema = 0; + } + if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || + type == TMQ_MSG_TYPE__WALINFO_RSP || + type == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code); @@ -558,7 +574,9 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* SEncoder encoder = {0}; tEncoderInit(&encoder, abuf, len); - if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || + type == TMQ_MSG_TYPE__WALINFO_RSP || + type == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { code = tEncodeMqDataRsp(&encoder, pRsp); } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { code = tEncodeSTaosxRsp(&encoder, pRsp); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index c7b1a816cd..b8b4505f9f 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -739,7 +739,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { } int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { - SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0, NULL); + SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0); if (pSW) { *num = pSW->nCols; tDeleteSchemaWrapper(pSW); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 476b1388d0..b28431013c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -314,25 +314,25 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int uint64_t nColData; if (tDecodeU64v(pCoder, &nColData) < 0) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } SColData colData = {0}; code = tDecodeColData(version, pCoder, &colData); if (code) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } if (colData.flag != HAS_VALUE) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } for (int32_t iRow = 0; iRow < colData.nVal; iRow++) { if (((TSKEY *)colData.pData)[iRow] < minKey || ((TSKEY *)colData.pData)[iRow] > maxKey) { code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } } @@ -340,14 +340,14 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int code = tDecodeColData(version, pCoder, &colData); if (code) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } } } else { uint64_t nRow; if (tDecodeU64v(pCoder, &nRow) < 0) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } for (int32_t iRow = 0; iRow < nRow; ++iRow) { @@ -356,7 +356,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int if (pRow->ts < minKey || pRow->ts > maxKey) { code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } } } @@ -369,6 +369,9 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int tEndDecode(pCoder); _exit: + if (code) { + vError("vgId:%d, %s:%d failed to vnodePreProcessSubmitTbData submit request since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { @@ -1889,7 +1892,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in len -= sizeof(SSubmitReq2Msg); SDecoder dc = {0}; tDecoderInit(&dc, pReq, len); - if (tDecodeSubmitReq(&dc, pSubmitReq) < 0) { + if (tDecodeSubmitReq(&dc, pSubmitReq, NULL) < 0) { code = TSDB_CODE_INVALID_MSG; goto _exit; } diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index ec041cba70..3bd6f4e64a 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -187,7 +187,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp terrno = TSDB_CODE_SUCCESS; if (NULL == pReq) { - if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { goto _end; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1060cbcffe..4f647a2e9c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3953,7 +3953,7 @@ FETCH_NEXT_BLOCK: QUERY_CHECK_NULL(pSubmit, code, lino, _end, terrno); qDebug("set %d/%d as the input submit block, %s", current + 1, totalBlocks, id); - if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < + if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver, NULL) < 0) { qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current, totalBlocks, id); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 161c5f7ca7..bc741b46ce 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -849,11 +849,6 @@ static int32_t physiDispatchCopy(const SDataDispatcherNode* pSrc, SDataDispatche return TSDB_CODE_SUCCESS; } -static int32_t physiInserterCopy(const SDataInserterNode* pSrc, SDataInserterNode* pDst) { - COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy); - return TSDB_CODE_SUCCESS; -} - static int32_t physiQueryInserterCopy(const SQueryInserterNode* pSrc, SQueryInserterNode* pDst) { COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy); CLONE_NODE_LIST_FIELD(pCols); @@ -1135,9 +1130,6 @@ int32_t nodesCloneNode(const SNode* pNode, SNode** ppNode) { case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: code = physiDispatchCopy((const SDataDispatcherNode*)pNode, (SDataDispatcherNode*)pDst); break; - //case QUERY_NODE_PHYSICAL_PLAN_INSERT: - // code = physiInserterCopy((const SDataInserterNode*)pNode, (SDataInserterNode*)pDst); - // break; case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: code = physiQueryInserterCopy((const SQueryInserterNode*)pNode, (SQueryInserterNode*)pDst); break; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 7006979216..9473e75642 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2000,7 +2000,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_INSERT: { SDataInserterNode* pSink = (SDataInserterNode*)pNode; destroyDataSinkNode((SDataSinkNode*)pSink); - taosMemoryFreeClear(pSink->pData); + taosMemFreeClear(pSink->pData); break; } case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 676aed4464..bd463bfd9d 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -403,3 +403,18 @@ end: } return code; } + +int32_t smlBuildOutputRaw(SQuery* handle, SHashObj* pVgHash) { + int32_t lino = 0; + int32_t code = 0; + + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot; + code = insBuildVgDataBlocks(pVgHash, pStmt->pVgDataBlocks, &pStmt->pDataBlocks, false); + TSDB_CHECK_CODE(code, lino, end); + + end: + if (code != 0) { + uError("%s failed at %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index ed1f498a32..1931f0803d 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -483,7 +483,7 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx return code; } -static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHash, SArray* pVgroupList, +static int32_t createVgroupDataCxt(int32_t vgId, SHashObj* pVgroupHash, SArray* pVgroupList, SVgroupDataCxt** pOutput) { SVgroupDataCxt* pVgCxt = taosMemoryCalloc(1, sizeof(SVgroupDataCxt)); if (NULL == pVgCxt) { @@ -495,7 +495,7 @@ static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHa return terrno; } - pVgCxt->vgId = pTableCxt->pMeta->vgId; + pVgCxt->vgId = vgId; int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES); if (TSDB_CODE_SUCCESS == code) { if (NULL == taosArrayPush(pVgroupList, &pVgCxt)) { @@ -642,7 +642,7 @@ int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, if (NULL == pp) { pp = taosHashGet(pBuildInfo->pVgroupHash, &vgId, sizeof(vgId)); if (NULL == pp) { - code = createVgroupDataCxt(pTbCtx, pBuildInfo->pVgroupHash, pBuildInfo->pVgroupList, &pVgCxt); + code = createVgroupDataCxt(vgId, pBuildInfo->pVgroupHash, pBuildInfo->pVgroupList, &pVgCxt); } else { pVgCxt = *(SVgroupDataCxt**)pp; } @@ -723,7 +723,7 @@ int32_t tbNum) { SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFun */ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks, bool isRebuild) { - SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES); if (NULL == pVgroupHash || NULL == pVgroupList) { taosHashCleanup(pVgroupHash); @@ -777,7 +777,7 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks, bool int32_t vgId = pTableCxt->pMeta->vgId; void** pp = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); if (NULL == pp) { - code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); + code = createVgroupDataCxt(vgId, pVgroupHash, pVgroupList, &pVgCxt); } else { pVgCxt = *(SVgroupDataCxt**)pp; } @@ -830,6 +830,7 @@ static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uin } static void destroyVgDataBlocks(void* p) { + if (p == NULL) return; SVgDataBlocks* pVg = p; taosMemoryFree(pVg->pData); taosMemoryFree(pVg); @@ -855,7 +856,6 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, if (TSDB_CODE_SUCCESS == code) { dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); - // uError("td23101 3vgId:%d, numEps:%d", src->vgId, dst->vg.epSet.numOfEps); } if (TSDB_CODE_SUCCESS == code) { code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); @@ -863,6 +863,9 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, if (TSDB_CODE_SUCCESS == code) { code = (NULL == taosArrayPush(pDataBlocks, &dst) ? terrno : TSDB_CODE_SUCCESS); } + if (TSDB_CODE_SUCCESS != code) { + destroyVgDataBlocks(dst); + } } if (append) { @@ -1074,3 +1077,33 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate end: return ret; } + +int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data) { + transformRawSSubmitTbData(data, pTableMeta->suid, pTableMeta->uid, pTableMeta->sversion); + SVgroupDataCxt* pVgCxt = NULL; + void** pp = taosHashGet(pVgroupHash, &pTableMeta->vgId, sizeof(pTableMeta->vgId)); + if (NULL == pp) { + int code = createVgroupDataCxt(pTableMeta->vgId, pVgroupHash, pVgroupList, &pVgCxt); + if (code != 0){ + return code; + } + } else { + pVgCxt = *(SVgroupDataCxt**)pp; + } + if (NULL == pVgCxt->pData->aSubmitTbData) { + pVgCxt->pData->aSubmitTbData = taosArrayInit(0, POINTER_BYTES); + pVgCxt->pData->raw = true; + if (NULL == pVgCxt->pData->aSubmitTbData) { + return terrno; + } + } + + // push data to submit, rebuild empty data for next submit + if (NULL == taosArrayPush(pVgCxt->pData->aSubmitTbData, &data)) { + return terrno; + } + + uTrace("add raw data to vgId:%d, len:%d", pTableMeta->vgId, *(int32_t*)data); + + return 0; +} diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 9513e90c50..316699bba1 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2839,7 +2839,8 @@ static int32_t createDataInserter(SPhysiPlanContext* pCxt, SVgDataBlocks* pBlock pInserter->numOfTables = pBlocks->numOfTables; pInserter->size = pBlocks->size; - TSWAP(pInserter->pData, pBlocks->pData); + pInserter->pData = pBlocks->pData; + pBlocks->pData = NULL; *pSink = (SDataSinkNode*)pInserter; return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index a031bc08de..ec9913d6ac 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1110,7 +1110,7 @@ _return: } int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType, void* param) { - int32_t msgSize = 0; + int32_t msgSize = 0; void *msg = NULL; int32_t code = 0; bool isCandidateAddr = false; @@ -1136,13 +1136,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_SUBMIT: case TDMT_VND_COMMIT: { msgSize = pTask->msgLen; - msg = taosMemoryCalloc(1, msgSize); - if (NULL == msg) { - SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(terrno); - } - - TAOS_MEMCPY(msg, pTask->msg, msgSize); + msg = pTask->msg; + pTask->msg = NULL; break; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index d7576005d7..b3b69d81c0 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -163,7 +163,7 @@ void* rpcMallocCont(int64_t contLen) { tTrace("malloc mem:%p size:%" PRId64, start, size); } - return start + sizeof(STransMsgHead); + return start + TRANS_MSG_OVERHEAD; } void rpcFreeCont(void* cont) { transFreeMsg(cont); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ba2d471ccf..f34b00bec5 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -859,6 +859,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT, "Replay is disabled TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_TABLE_QUALIFIED, "No table qualified for query") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_NEED_REBALANCE, "No need rebalance") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_STATUS, "Invalid status, please subscribe topic first") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_DATA, "Invalid data use here") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_RAW_DATA_SPLIT, "Split submit data for rawdata") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") diff --git a/tests/army/whole/checkErrorCode.py b/tests/army/whole/checkErrorCode.py index b2aa1fce1b..1a14269f54 100644 --- a/tests/army/whole/checkErrorCode.py +++ b/tests/army/whole/checkErrorCode.py @@ -61,7 +61,8 @@ ignoreCodes = [ '0x80003107', '0x80003108', '0x80003109', '0x80003110', '0x80003111', '0x80003112', '0x80003250', '0x80004003', '0x80004004', '0x80004005', '0x80004006', '0x80004007', '0x80004008', '0x80004009', '0x80004010', '0x80004011', '0x80004012', '0x80004013', '0x80004014', '0x80004015', '0x80004016', '0x80004102', '0x80004103', '0x80004104', '0x80004105', '0x80004106', '0x80004107', '0x80004108', '0x80004109', '0x80005100', - '0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666', '0x80000237'] + '0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666', '0x80000237', '0x80004018', + '0x80004019'] class TDTestCase(TBase): diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index accd57d739..26cc122021 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -455,6 +455,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_td33504.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5776.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5906.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py diff --git a/tests/system-test/7-tmq/taosx-performance.py b/tests/system-test/7-tmq/taosx-performance.py new file mode 100755 index 0000000000..5f7c6c3e07 --- /dev/null +++ b/tests/system-test/7-tmq/taosx-performance.py @@ -0,0 +1,280 @@ +import taos +import sys +import time +import socket +import os +import threading + +sys.path.append("../../pytest") + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * + +sys.path.append("./7-tmq") +from tmqCommon import * + +tdDnodes1 = TDDnodes() +tdDnodes2 = TDDnodes() +vgroups = 1 +tables = 1 +rows_of_each_table = 1 +path = "./taosx_perf" +dnode1 = "./dnode1" +dnode2 = "./dnode2" +cfg = "sim/dnode1/cfg" +taosx = "taosx" +taosd = "taosd" +taosxLog = "taosx.log" +taosxTimeout = 2 +timeCost = [] +speedupStr = [] +insertJson = '''{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 20, + "thread_bind_vgroup": "yes", + "thread_count": 20, + "create_table_thread_count": 16, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "num_of_records_per_req": 10, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "continue_if_fail": "no", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "vgroups": 8, + "precision": "ms", + "WAL_RETENTION_PERIOD": 864000 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 10000, + "childtable_prefix": "d", + "auto_create_table": "yes", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 1000, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 1, + "insert_interval": 0, + "partial_col_num": 0, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "generate_row_rule": 2, + "columns": [ + {"type": "TINYINT", "name": "current", "max": 128, "min": 1 }, + { "type": "BOOL", "name": "phaseewe" }, + { "type": "BINARY", "name": "str", "len":16374, + "values": [ + "{kkey00000k: kvalue00000k, kkey00001k: kvalue00001k, kkey00002k: kvalue00002k, kkey00003k: kvalue00003k, kkey00004k: kvalue00004k, kkey00005k: kvalue00005k, kkey00006k: kvalue00006k, kkey00007k: kvalue00007k, kkey00008k: kvalue00008k, kkey00009k: kvalue00009k, kkey00010k: kvalue00010k, kkey00011k: kvalue00011k, kkey00012k: kvalue00012k, kkey00013k: kvalue00013k, kkey00014k: kvalue00014k, kkey00015k: kvalue00015k, kkey00016k: kvalue00016k, kkey00017k: kvalue00017k, kkey00018k: kvalue00018k, kkey00019k: kvalue00019k, kkey00020k: kvalue00020k, kkey00021k: kvalue00021k, kkey00022k: kvalue00022k, kkey00023k: kvalue00023k, kkey00024k: kvalue00024k, kkey00025k: kvalue00025k, kkey00026k: kvalue00026k, kkey00027k: kvalue00027k, kkey00028k: kvalue00028k, kkey00029k: kvalue00029k, kkey00030k: kvalue00030k, kkey00031k: kvalue00031k, kkey00032k: kvalue00032k, kkey00033k: kvalue00033k, kkey00034k: kvalue00034k, kkey00035k: kvalue00035k, kkey00036k: kvalue00036k, kkey00037k: kvalue00037k, kkey00038k: kvalue00038k, kkey00039k: kvalue00039k, kkey00040k: kvalue00040k, kkey00041k: kvalue00041k, kkey00042k: kvalue00042k, kkey00043k: kvalue00043k, kkey00044k: kvalue00044k, kkey00045k: kvalue00045k, kkey00046k: kvalue00046k, kkey00047k: kvalue00047k, kkey00048k: kvalue00048k, kkey00049k: kvalue00049k, kkey00050k: kvalue00050k, kkey00051k: kvalue00051k, kkey00052k: kvalue00052k, kkey00053k: kvalue00053k, kkey00054k: kvalue00054k, kkey00055k: kvalue00055k, kkey00056k: kvalue00056k, kkey00057k: kvalue00057k, kkey00058k: kvalue00058k, kkey00059k: kvalue00059k, kkey00060k: kvalue00060k, kkey00061k: kvalue00061k, kkey00062k: kvalue00062k, kkey00063k: kvalue00063k, kkey00064k: kvalue00064k, kkey00065k: kvalue00065k, kkey00066k: kvalue00066k, kkey00067k: kvalue00067k, kkey00068k: kvalue00068k, kkey00069k: kvalue00069k, kkey00070k: kvalue00070k, kkey00071k: kvalue00071k, kkey00072k: kvalue00072k, kkey00073k: kvalue00073k, kkey00074k: kvalue00074k, kkey00075k: kvalue00075k, kkey00076k: kvalue00076k, kkey00077k: kvalue00077k, kkey00078k: kvalue00078k, kkey00079k: kvalue00079k, kkey00080k: kvalue00080k, kkey00081k: kvalue00081k, kkey00082k: kvalue00082k, kkey00083k: kvalue00083k, kkey00084k: kvalue00084k, kkey00085k: kvalue00085k, kkey00086k: kvalue00086k, kkey00087k: kvalue00087k, kkey00088k: kvalue00088k, kkey00089k: kvalue00089k, kkey00090k: kvalue00090k, kkey00091k: kvalue00091k, kkey00092k: kvalue00092k, kkey00093k: kvalue00093k, kkey00094k: kvalue00094k, kkey00095k: kvalue00095k, kkey00096k: kvalue00096k, kkey00097k: kvalue00097k, kkey00098k: kvalue00098k, kkey00099k: kvalue00099k, kkey00100k: kvalue00100k, kkey00101k: kvalue00101k, kkey00102k: kvalue00102k, kkey00103k: kvalue00103k, kkey00104k: kvalue00104k, kkey00105k: kvalue00105k, kkey00106k: kvalue00106k, kkey00107k: kvalue00107k, kkey00108k: kvalue00108k, kkey00109k: kvalue00109k, kkey00110k: kvalue00110k, kkey00111k: kvalue00111k, kkey00112k: kvalue00112k, kkey00113k: kvalue00113k, kkey00114k: kvalue00114k, kkey00115k: kvalue00115k, kkey00116k: kvalue00116k, kkey00117k: kvalue00117k, kkey00118k: kvalue00118k, kkey00119k: kvalue00119k, kkey00120k: kvalue00120k, kkey00121k: kvalue00121k, kkey00122k: kvalue00122k, kkey00123k: kvalue00123k, kkey00124k: kvalue00124k, kkey00125k: kvalue00125k, kkey00126k: kvalue00126k, kkey00127k: kvalue00127k, kkey00128k: kvalue00128k, kkey00129k: kvalue00129k, kkey00130k: kvalue00130k, kkey00131k: kvalue00131k, kkey00132k: kvalue00132k, kkey00133k: kvalue00133k, kkey00134k: kvalue00134k, kkey00135k: kvalue00135k, kkey00136k: kvalue00136k, kkey00137k: kvalue00137k, kkey00138k: kvalue00138k, kkey00139k: kvalue00139k, kkey00140k: kvalue00140k, kkey00141k: kvalue00141k, kkey00142k: kvalue00142k, kkey00143k: kvalue00143k, kkey00144k: kvalue00144k, kkey00145k: kvalue00145k, kkey00146k: kvalue00146k, kkey00147k: kvalue00147k, kkey00148k: kvalue00148k, kkey00149k: kvalue00149k, kkey00150k: kvalue00150k, kkey00151k: kvalue00151k, kkey00152k: kvalue00152k, kkey00153k: kvalue00153k, kkey00154k: kvalue00154k, kkey00155k: kvalue00155k, kkey00156k: kvalue00156k, kkey00157k: kvalue00157k, kkey00158k: kvalue00158k, kkey00159k: kvalue00159k, kkey00160k: kvalue00160k, kkey00161k: kvalue00161k, kkey00162k: kvalue00162k, kkey00163k: kvalue00163k, kkey00164k: kvalue00164k, kkey00165k: kvalue00165k, kkey00166k: kvalue00166k, kkey00167k: kvalue00167k, kkey00168k: kvalue00168k, kkey00169k: kvalue00169k, kkey00170k: kvalue00170k, kkey00171k: kvalue00171k, kkey00172k: kvalue00172k, kkey00173k: kvalue00173k, kkey00174k: kvalue00174k, kkey00175k: kvalue00175k, kkey00176k: kvalue00176k, kkey00177k: kvalue00177k, kkey00178k: kvalue00178k, kkey00179k: kvalue00179k, kkey00180k: kvalue00180k, kkey00181k: kvalue00181k, kkey00182k: kvalue00182k, kkey00183k: kvalue00183k, kkey00184k: kvalue00184k, kkey00185k: kvalue00185k, kkey00186k: kvalue00186k, kkey00187k: kvalue00187k, kkey00188k: kvalue00188k, kkey00189k: kvalue00189k, kkey00190k: kvalue00190k, kkey00191k: kvalue00191k, kkey00192k: kvalue00192k, kkey00193k: kvalue00193k, kkey00194k: kvalue00194k, kkey00195k: kvalue00195k, kkey00196k: kvalue00196k, kkey00197k: kvalue00197k, kkey00198k: kvalue00198k, kkey00199k: kvalue00199k}", + "NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL", + "NULerL","NdrerULL","NU232LL","NUL23L","NU23LL","NU23LL","NUL23L","NUL32L" + ] + }, + { "type": "BIGINT", "name": "cnt", "max" : 2563332323232, "min":1 }, + { "type": "DOUBLE", "name": "phase", "max": 1000, "min": 0 } + ], + "tags": [ + {"type": "TINYINT", "name": "groupid", "max": 10, "min": 1}, + {"type": "BINARY", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +}''' + + +def initEnv(): + updatecfgDict2 = {'debugFlag': 131, 'serverPort': 7030} + tdDnodes2.init(dnode2) + tdDnodes2.deploy(1, updatecfgDict2) + tdDnodes2.start(1) + + updatecfgDict1 = {'debugFlag': 131, 'serverPort': 6030} + tdDnodes1.init(dnode1) + tdDnodes1.deploy(1, updatecfgDict1) + tdDnodes1.start(1) + + with open('taosx-performance.json', 'w') as file: + file.write(insertJson) + + changeVgroups = f"sed -i '/vgroups/c\ \"vgroups\": {vgroups},' taosx-performance.json" + os.system(changeVgroups) + + changeTables = f"sed -i '/childtable_count/c\ \"childtable_count\": {tables},' taosx-performance.json" + os.system(changeTables) + + changeRows = f"sed -i '/insert_rows/c\ \"insert_rows\": {rows_of_each_table},' taosx-performance.json" + os.system(changeRows) + os.system("taosBenchmark -f taosx-performance.json") + +def stopTaosd(str): + psCmd = f"ps -ef | grep -w taosd | grep -v grep | grep {str}" + " | awk '{print $2}' | xargs" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + print(f"kill taosd pid={processID}") + if processID: + cmd = f"kill -9 {processID}" + os.system(cmd) + +def cleanDb(): + dropTopic = f"{taosd}/bin/taos -c {dnode1}/{cfg} -s \"drop topic if exists test\"" + print("dropTopic:%s" % dropTopic) + os.system(dropTopic) + + dropDb = f"{taosd}/bin/taos -c {dnode2}/{cfg} -s \"drop database if exists test\"" + print("dropDb:%s" % dropDb) + os.system(dropDb) + + createDb = f"{taosd}/bin/taos -c {dnode2}/{cfg} -s \"create database test vgroups {vgroups}\"" + print("createDb:%s" % createDb) + os.system(createDb) + + +def restartTaosd(): + cmd1 = f"{taosd}/bin/taosd -c {dnode1}/{cfg} > /dev/null 2>&1 &" + cmd2 = f"{taosd}/bin/taosd -c {dnode2}/{cfg} > /dev/null 2>&1 &" + print("start taosd1 :%s" % cmd1) + print("start taosd2 :%s" % cmd2) + os.system(cmd1) + os.system(cmd2) + + +def runTaosx(): + cmd = f"{taosx} run -f \"tmq://root:taosdata@localhost:6030/test?group.id=taosx-new-`date +%s`&timeout={taosxTimeout}s&experimental.snapshot.enable=false&auto.offset.reset=earliest&prefer=raw&libraryPath={taosd}/lib/libtaos.so\" -t \"taos://root:taosdata@localhost:7030/test?libraryPath={taosd}/lib/libtaos.so\" > {taosxLog}" + print("run taosx:%s" % cmd) + os.system(cmd) + + +def parseArgs(argv): + if len(argv) < 6: + print( + "Usage: python3 taosx-performance.py path_to_taosx path_to_taosd init vgroups tables rows_of_each_table [path]") + sys.exit(1) + + global taosx + global taosd + global init + global vgroups + global tables + global rows_of_each_table + global path + global dnode1 + global dnode2 + taosx = argv[0] + taosd = argv[1] + init = argv[2] + vgroups = argv[3] + tables = argv[4] + rows_of_each_table = argv[5] + if len(argv) == 7: + path = argv[6] + + if not os.path.isdir(path): + mkCmd = f"mkdir {path}" + os.system(mkCmd) + # os.system("git clone https://github.com/brendangregg/FlameGraph.git taosx-perf") +def getCost(): + costCmd = f"cat {taosxLog}| grep 'time cost'" + "| awk '{print $3}' | xargs" + cost = subprocess.check_output(costCmd, shell=True).decode("utf-8").strip() + return cost + +def runOnce(argv): + parseArgs(argv) + + os.chdir(path) + print("current dir:" + os.getcwd()) + + stopTaosd("dnode2") + stopTaosd("dnode1") + time.sleep(2) + + if init == "true": + initEnv() + else: + restartTaosd() + + cleanDb() + runTaosx() + + cost = getCost() + print("cost:%s" % cost) + timeCost.append(cost) + +''' +python3 taosx-performance.py once path_to_taosx path_to_taosd init vgroups tables rows_of_each_table [path] +''' +paras = [ + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 1, 5000, 200, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 1, 5000, 200, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 1, 5000, 1000, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 1, 5000, 1000, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 1, 5000, 2000, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 1, 5000, 2000, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 2, 10000, 100, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 2, 10000, 100, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 2, 10000, 500, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 2, 10000, 500, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 2, 10000, 1000, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 2, 10000, 1000, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 4, 20000, 500, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 4, 20000, 500, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 8, 40000, 250, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 8, 40000, 250, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 16, 100000, 100, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 16, 100000, 100, "/tmp/taosx_perf"), +] +if __name__ == "__main__": + print("run performance start") + + once = sys.argv[1] + if once == "once": + runOnce(sys.argv[2:]) + else: + for i in range(len(paras)): + runOnce(paras[i]) + if i % 2 == 1 : + print(f"opti cost:{float(timeCost[0]) - taosxTimeout}") + print(f"old cost:{float(timeCost[1]) - taosxTimeout}") + tmp = str(paras[i]) + f" speedup:{(float(timeCost[1]) - taosxTimeout)/(float(timeCost[0]) - taosxTimeout)}" + speedupStr.append(tmp) + print(tmp + "\n\n\n") + timeCost.clear() + + print("performance result:\n" + str(speedupStr)) + tdLog.info("run performance end") diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 5047ada1d1..845e7229a8 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -17,6 +17,8 @@ from tmqCommon import * class TDTestCase: updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + clientCfgDict = {'debugFlag': 135, 'asynclog': 0} + updatecfgDict["clientCfg"] = clientCfgDict def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") @@ -65,7 +67,7 @@ class TDTestCase: tdSql.checkData(1, 5, "sttb4") tdSql.query("select * from stt order by ts") - tdSql.checkRows(3) + tdSql.checkRows(5) tdSql.checkData(0, 1, 1) tdSql.checkData(2, 1, 21) tdSql.checkData(0, 2, 2) @@ -96,7 +98,7 @@ class TDTestCase: tdSql.checkData(1, 5, "sttb4") tdSql.query("select * from stt order by ts") - tdSql.checkRows(3) + tdSql.checkRows(5) tdSql.checkData(0, 1, 1) tdSql.checkData(2, 1, 21) tdSql.checkData(0, 2, 2) @@ -256,6 +258,17 @@ class TDTestCase: return + def checkWalMultiVgroupsRawData(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -sv 3 -dv 5 -raw'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkData() + self.checkDropData(False) + + return + def checkWalMultiVgroupsWithDropTable(self): buildPath = tdCom.getBuildPath() cmdStr = '%s/build/bin/tmq_taosx_ci -sv 3 -dv 5 -d'%(buildPath) @@ -681,6 +694,7 @@ class TDTestCase: self.checkSnapshot1VgroupTable() self.checkWalMultiVgroups() + self.checkWalMultiVgroupsRawData() self.checkSnapshotMultiVgroups() self.checkWalMultiVgroupsWithDropTable() diff --git a/tests/system-test/7-tmq/tmq_ts-5776.py b/tests/system-test/7-tmq/tmq_ts-5776.py new file mode 100644 index 0000000000..738d69701f --- /dev/null +++ b/tests/system-test/7-tmq/tmq_ts-5776.py @@ -0,0 +1,39 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_ts5776'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + return + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index d1c049ef1e..d73f91aad3 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -6,6 +6,7 @@ add_executable(tmq_taosx_ci tmq_taosx_ci.c) add_executable(tmq_ts5466 tmq_ts5466.c) add_executable(tmq_td32526 tmq_td32526.c) add_executable(tmq_td32187 tmq_td32187.c) +add_executable(tmq_ts5776 tmq_ts5776.c) add_executable(tmq_td32471 tmq_td32471.c) add_executable(tmq_write_raw_test tmq_write_raw_test.c) add_executable(write_raw_block_test write_raw_block_test.c) @@ -87,6 +88,13 @@ target_link_libraries( PUBLIC common PUBLIC os ) +target_link_libraries( + tmq_ts5776 + PUBLIC ${TAOS_LIB} + PUBLIC util + PUBLIC common + PUBLIC os +) target_link_libraries( tmq_taosx_ci PUBLIC ${TAOS_LIB} diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 117f9fa2e1..bc4a79bf12 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -34,6 +34,7 @@ typedef struct { int dstVgroups; char dir[256]; bool btMeta; + bool rawData; } Config; Config g_conf = {0}; @@ -424,6 +425,15 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); + pRes = + taos_query(pConn, + "insert into stt1 values(now + 322s, 3, 2, 'stt1')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "insert into stt1 values(now + 2s, 3, 2, 'stt1') stt3 using stt tags(23, \"stt3\", true) values(now + " @@ -436,6 +446,15 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); + pRes = + taos_query(pConn, + "insert into stt1 values(now + 442s, 3, 2, 'stt1')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; } @@ -633,6 +652,9 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); tmq_conf_set(conf, "msg.consume.excluded", "1"); + if (g_conf.rawData) { + tmq_conf_set(conf, "msg.consume.rawdata", "1"); + } // tmq_conf_set(conf, "session.timeout.ms", "1000000"); // tmq_conf_set(conf, "max.poll.interval.ms", "20000"); @@ -1238,6 +1260,8 @@ int main(int argc, char* argv[]) { g_conf.meta = 1; } else if (strcmp(argv[i], "-bt") == 0) { g_conf.btMeta = true; + } else if (strcmp(argv[i], "-raw") == 0) { + g_conf.rawData = true; } } diff --git a/utils/test/c/tmq_ts5776.c b/utils/test/c/tmq_ts5776.c new file mode 100644 index 0000000000..c50809bccd --- /dev/null +++ b/utils/test/c/tmq_ts5776.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "cJSON.h" +#include "taos.h" +#include "tmsg.h" +#include "types.h" + +static TAOS* use_db() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return NULL; + } + + TAOS_RES* pRes = taos_query(pConn, "use db_dst"); + if (taos_errno(pRes) != 0) { + printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); + return NULL; + } + taos_free_result(pRes); + return pConn; +} + +static void msg_process(TAOS_RES* msg) { + printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); + printf("db: %s\n", tmq_get_db_name(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); + TAOS* pConn = use_db(); + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META || tmq_get_res_type(msg) == TMQ_RES_METADATA) { + char* result = tmq_get_json_meta(msg); + printf("meta result: %s\n", result); + tmq_free_json_meta(result); + } + + tmq_raw_data raw = {0}; + tmq_get_raw(msg, &raw); + printf("write raw data type: %d\n", raw.raw_type); + int32_t ret = tmq_write_raw(pConn, raw); + printf("write raw data: %s\n", tmq_err2str(ret)); + ASSERT(ret == 0); + + tmq_free_raw(raw); + taos_close(pConn); +} + +int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query( + pConn, + "insert into ct3 using st1(t1) tags(3000) values(1626006833600, 5, 6, 'c') ct1 using st1(t1) tags(2000) values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, " + "'ddd') ct0 using st1 tags(1000, \"ttt\", true) values(1626006833603, 4, 3, 'hwj') ct1 using st1(t1) tags(2000) values(now+5s, 23, 32, 's21ds')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query( + pConn, + "insert into ct3 values(1626006839602, 5, 6, 'c')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query( + pConn, + "insert into ct3 using st1(t1) tags(3000) values(1626006833602, 5, 6, 'c') ct1 values(1626006833611, 2, 3, 'sds') (1626006833612, 4, 5, " + "'ddd') ct10 using st1 tags(1000, \"ttt\", true) values(1626006833603, 4, 3, 'hwj')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + return 0; +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_dst"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_dst vgroups 1 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop topic if exists topic_db"); + if (taos_errno(pRes) != 0) { + printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists db_src"); + if (taos_errno(pRes) != 0) { + printf("error in drop db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_src vgroups 1 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use db_src"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + buildDatabase(pConn, pRes); + + taos_close(pConn); + return 0; +} + +int32_t create_topic() { + printf("create topic\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "create topic topic_db with meta as database db_src"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.rawdata", "1"); + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "topic_db"); + return topic_list; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000); + if (tmqmessage) { + cnt++; + msg_process(tmqmessage); + taos_free_result(tmqmessage); + } else { + break; + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void check_result(){ + TAOS* taos = use_db(); + TAOS_RES *res = taos_query(taos, "select * from st1"); + int code = taos_errno(res); + assert(code == 0); + int affectedRows = taos_affected_rows(res); + printf("affected rows %d\n", affectedRows); + assert(affectedRows == 10); + taos_free_result(res); + taos_close(taos); + +} +int main(int argc, char* argv[]) { + if (init_env() < 0) { + return -1; + } + create_topic(); + + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + tmq_list_destroy(topic_list); +}