diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index bb18516ba4..ccd60df19a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -109,6 +109,13 @@ cat("${TD_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) # cJson cat("${TD_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +# xz +#cat("${TD_SUPPORT_DIR}/xz_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + +#lzma2 +cat("${TD_SUPPORT_DIR}/lzma_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + + if (${BUILD_CONTRIB}) if(${BUILD_WITH_ROCKSDB}) cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -127,6 +134,8 @@ else() endif() endif() +#cat("${TD_SUPPORT_DIR}/zstd_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + #libuv if(${BUILD_WITH_UV}) cat("${TD_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -254,6 +263,13 @@ target_include_directories( ) unset(CMAKE_PROJECT_INCLUDE_BEFORE) +# add_subdirectory(xz EXCLUDE_FROM_ALL) +# target_include_directories( +# xz +# PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/xz +# PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/xz +# ) + # leveldb if(${BUILD_WITH_LEVELDB}) option(LEVELDB_BUILD_TESTS "" OFF) @@ -388,7 +404,6 @@ endif() if(${BUILD_WITH_S3}) INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include) MESSAGE("build with s3: ${BUILD_WITH_S3}") - # cos elseif(${BUILD_WITH_COS}) if(${TD_LINUX}) @@ -427,6 +442,7 @@ if(${BUILD_PTHREAD}) target_link_libraries(pthread INTERFACE libpthreadVC3) endif() + # jemalloc if(${JEMALLOC_ENABLED}) include(ExternalProject) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 73399d99d1..ed23290be4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3942,6 +3942,7 @@ int32_t tDecodeMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp); void tDeleteMqMetaRsp(SMqMetaRsp *pRsp); #define MQ_DATA_RSP_VERSION 100 + typedef struct { SMqRspHead head; STqOffsetVal reqOffset; @@ -3953,33 +3954,27 @@ typedef struct { SArray* blockData; SArray* blockTbName; SArray* blockSchema; - int64_t sleepTime; +} SMqDataRspCommon; + +typedef struct { + SMqDataRspCommon common; + int64_t sleepTime; } SMqDataRsp; -int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); -int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp, int8_t dataVersion); -void tDeleteMqDataRsp(SMqDataRsp* pRsp); +int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const void* pRsp); +int32_t tDecodeMqDataRsp(SDecoder* pDecoder, void* pRsp); +void tDeleteMqDataRsp(void* pRsp); typedef struct { - SMqRspHead head; - STqOffsetVal reqOffset; - STqOffsetVal rspOffset; - int32_t blockNum; - int8_t withTbName; - int8_t withSchema; - SArray* blockDataLen; - SArray* blockData; - SArray* blockTbName; - SArray* blockSchema; - // the following attributes are extended from SMqDataRsp - int32_t createTableNum; - SArray* createTableLen; - SArray* createTableReq; + SMqDataRspCommon common; + int32_t createTableNum; + SArray* createTableLen; + SArray* createTableReq; } STaosxRsp; -int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const STaosxRsp* pRsp); -int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, STaosxRsp* pRsp, int8_t dateVersion); -void tDeleteSTaosxRsp(STaosxRsp* pRsp); +int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const void* pRsp); +int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, void* pRsp); +void tDeleteSTaosxRsp(void* pRsp); typedef struct { SMqRspHead head; diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 93e0064192..cb616f7afc 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -41,5 +41,6 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg); int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode); int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode); +void tqSetRestoreVersionInfo(SStreamTask* pTask); #endif // TDENGINE_TQ_COMMON_H diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 7ae393f5a6..37d5f3f0b6 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -152,7 +152,8 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int32_t msgBufLen); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); -int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength); +int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD *fields, + int numFields, bool needChangeLength, char* errstr, int32_t errstrLen); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index acd9ac52ea..75e6fc87e7 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -33,8 +33,16 @@ extern "C" { const char* tstrerror(int32_t err); const char* terrstr(); +#define ERR_MSG_LEN 256 + +char* taosGetErrMsgReturn(); +char* taosGetErrMsg(); int32_t* taosGetErrno(); #define terrno (*taosGetErrno()) +#define terrMsg (taosGetErrMsg()) + +#define SET_ERROR_MSG(MSG, ...) \ + snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__) #define TSDB_CODE_SUCCESS 0 #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 6c3603b4e0..9507472df0 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -221,7 +221,11 @@ typedef struct { SSchemaWrapper schema; int32_t resIter; SReqResultInfo resInfo; - SMqDataRsp rsp; +} SMqRspObjCommon; + +typedef struct { + SMqRspObjCommon common; + SMqDataRsp rsp; } SMqRspObj; typedef struct { @@ -233,14 +237,8 @@ typedef struct { } SMqMetaRspObj; typedef struct { - int8_t resType; - char topic[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int32_t vgId; - SSchemaWrapper schema; - int32_t resIter; - SReqResultInfo resInfo; - STaosxRsp rsp; + SMqRspObjCommon common; + STaosxRsp rsp; } SMqTaosxRspObj; typedef struct SReqRelInfo { @@ -320,7 +318,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; - return (SReqResultInfo*)&msg->resInfo; + return (SReqResultInfo*)&msg->common.resInfo; } SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 19798fa38b..ba7f65c52b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -343,12 +343,12 @@ void taos_free_result(TAOS_RES *res) { } else if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj *pRsp = (SMqTaosxRspObj *)res; tDeleteSTaosxRsp(&pRsp->rsp); - doFreeReqResultInfo(&pRsp->resInfo); + doFreeReqResultInfo(&pRsp->common.resInfo); taosMemoryFree(pRsp); } else if (TD_RES_TMQ(res)) { SMqRspObj *pRsp = (SMqRspObj *)res; tDeleteMqDataRsp(&pRsp->rsp); - doFreeReqResultInfo(&pRsp->resInfo); + doFreeReqResultInfo(&pRsp->common.resInfo); taosMemoryFree(pRsp); } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res; @@ -417,7 +417,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SMqRspObj *msg = ((SMqRspObj *)res); SReqResultInfo *pResultInfo; - if (msg->resIter == -1) { + if (msg->common.resIter == -1) { pResultInfo = tmqGetNextResInfo(res, true); } else { pResultInfo = tmqGetCurResInfo(res); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 84a2bf3e03..eb33263286 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1486,7 +1486,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); - code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false); + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -1570,7 +1570,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); - code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false); + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -1608,6 +1608,7 @@ static void* getRawDataFromRes(void* pRetrieve) { static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { if (taos == NULL || data == NULL) { terrno = TSDB_CODE_INVALID_PARA; + SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data); return terrno; } int32_t code = TSDB_CODE_SUCCESS; @@ -1620,17 +1621,24 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { terrno = TSDB_CODE_SUCCESS; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { + SET_ERROR_MSG("pRequest is NULL"); return terrno; } uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); pRequest->syncQuery = true; - rspObj.resIter = -1; - rspObj.resType = RES_TYPE__TMQ; + rspObj.common.resIter = -1; + rspObj.common.resType = RES_TYPE__TMQ; + int8_t dataVersion = *(int8_t*)data; + if (dataVersion >= MQ_DATA_RSP_VERSION){ + data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t)); + dataLen -= sizeof(int8_t) + sizeof(int32_t); + } tDecoderInit(&decoder, data, dataLen); - code = tDecodeMqDataRsp(&decoder, &rspObj.rsp, *(int8_t*)data); + code = tDecodeMqDataRsp(&decoder, &rspObj.rsp); if (code != 0) { + SET_ERROR_MSG("decode mq data rsp failed"); code = TSDB_CODE_INVALID_MSG; goto end; } @@ -1643,6 +1651,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get handle failed"); goto end; } @@ -1655,17 +1664,19 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { pQuery = smlInitHandle(); if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + SET_ERROR_MSG("init sml handle failed"); goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - while (++rspObj.resIter < rspObj.rsp.blockNum) { - void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); - if (!rspObj.rsp.withSchema) { + while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { + void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); + if (!rspObj.rsp.common.withSchema) { goto end; } - const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); + const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter); if (!tbName) { + SET_ERROR_MSG("block tbname is null"); code = TSDB_CODE_TMQ_INVALID_MSG; goto end; } @@ -1681,12 +1692,14 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { // continue; // } if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get table:%s meta failed", tbName); goto end; } SVgroupInfo vg; code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get table:%s vgroup failed", tbName); goto end; } @@ -1695,9 +1708,11 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); if (fields == NULL) { + SET_ERROR_MSG("calloc fields failed"); + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } for (int i = 0; i < pSW->nCols; i++) { @@ -1706,16 +1721,18 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } void* rawData = getRawDataFromRes(pRetrieve); - code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true); + char err[ERR_MSG_LEN] = {0}; + code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN); taosMemoryFree(fields); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("table:%s, err:%s", tbName, err); goto end; } - taosMemoryFreeClear(pTableMeta); } code = smlBuildOutput(pQuery, pVgHash); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("sml build output failed"); goto end; } @@ -1737,6 +1754,7 @@ end: static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { if (taos == NULL || data == NULL) { terrno = TSDB_CODE_INVALID_PARA; + SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data); return terrno; } int32_t code = TSDB_CODE_SUCCESS; @@ -1750,16 +1768,24 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) terrno = TSDB_CODE_SUCCESS; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { + SET_ERROR_MSG("pRequest is NULL"); return terrno; } uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); pRequest->syncQuery = true; - rspObj.resIter = -1; - rspObj.resType = RES_TYPE__TMQ_METADATA; + rspObj.common.resIter = -1; + rspObj.common.resType = RES_TYPE__TMQ_METADATA; + + int8_t dataVersion = *(int8_t*)data; + if (dataVersion >= MQ_DATA_RSP_VERSION){ + data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t)); + dataLen -= sizeof(int8_t) + sizeof(int32_t); + } tDecoderInit(&decoder, data, dataLen); - code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp, *(int8_t*)data); + code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp); if (code != 0) { + SET_ERROR_MSG("decode mq taosx data rsp failed"); code = TSDB_CODE_INVALID_MSG; goto end; } @@ -1772,6 +1798,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get handle failed"); goto end; } @@ -1783,20 +1810,22 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) pQuery = smlInitHandle(); if (pQuery == NULL) { + SET_ERROR_MSG("init sml handle failed"); code = TSDB_CODE_OUT_OF_MEMORY; goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.blockNum); - while (++rspObj.resIter < rspObj.rsp.blockNum) { - void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); - if (!rspObj.rsp.withSchema) { + uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); + while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { + void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); + if (!rspObj.rsp.common.withSchema) { goto end; } - const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); + const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter); if (!tbName) { + SET_ERROR_MSG("block tbname is null"); code = TSDB_CODE_TMQ_INVALID_MSG; goto end; } @@ -1818,6 +1847,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tDecoderClear(&decoderTmp); tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); code = TSDB_CODE_TMQ_INVALID_MSG; + SET_ERROR_MSG("decode create table:%s req failed", tbName); goto end; } @@ -1825,6 +1855,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) code = TSDB_CODE_TSC_INVALID_VALUE; tDecoderClear(&decoderTmp); tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); + SET_ERROR_MSG("create table req type is not child table: %s, type: %d", tbName, pCreateReq.type); goto end; } if (strcmp(tbName, pCreateReq.name) == 0) { @@ -1841,6 +1872,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) SVgroupInfo vg; code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get table:%s vgroup failed", tbName); goto end; } @@ -1854,6 +1886,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) // continue; // } if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("cata log get table:%s meta failed", tbName); goto end; } @@ -1867,9 +1900,11 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter); TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); if (fields == NULL) { + SET_ERROR_MSG("calloc fields failed"); + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } for (int i = 0; i < pSW->nCols; i++) { @@ -1878,12 +1913,13 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } void* rawData = getRawDataFromRes(pRetrieve); - code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true); + char err[ERR_MSG_LEN] = {0}; + code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN); taosMemoryFree(fields); if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("table:%s, err:%s", tbName, err); goto end; } - taosMemoryFreeClear(pTableMeta); } code = smlBuildOutput(pQuery, pVgHash); @@ -1946,6 +1982,64 @@ char* tmq_get_json_meta(TAOS_RES* res) { void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } +static int32_t getOffSetLen(const void *rsp){ + const SMqDataRspCommon *pRsp = rsp; + SEncoder coder = {0}; + tEncoderInit(&coder, NULL, 0); + if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1; + if (tEncodeSTqOffsetVal(&coder, &pRsp->rspOffset) < 0) return -1; + int32_t pos = coder.pos; + tEncoderClear(&coder); + return pos; +} + +typedef int32_t __encode_func__(SEncoder *pEncoder, const void *pRsp); + +static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, void* rspObj, tmq_raw_data* raw){ + int32_t len = 0; + int32_t code = 0; + SEncoder encoder = {0}; + void* buf = NULL; + tEncodeSize(encodeFunc, rspObj, len, code); + if (code < 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + len += sizeof(int8_t) + sizeof(int32_t); + buf = taosMemoryCalloc(1, len); + if(buf == NULL){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAILED; + } + tEncoderInit(&encoder, buf, len); + if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + int32_t offsetLen = getOffSetLen(rspObj); + if(offsetLen <= 0){ + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + if (tEncodeI32(&encoder, offsetLen) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + if(encodeFunc(&encoder, rspObj) < 0){ + terrno = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + tEncoderClear(&encoder); + + raw->raw = buf; + raw->raw_len = len; + return 0; +FAILED: + tEncoderClear(&encoder); + taosMemoryFree(buf); + return terrno; +} + int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { if (!raw || !res) { terrno = TSDB_CODE_INVALID_PARA; @@ -1959,42 +2053,19 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { uDebug("tmq get raw type meta:%p", raw); } else if (TD_RES_TMQ(res)) { SMqRspObj* rspObj = ((SMqRspObj*)res); - - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeMqDataRsp, &rspObj->rsp, len, code); - if (code < 0) { - return -1; + if(encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw) != 0){ + uError("tmq get raw type error:%d", terrno); + return terrno; } - - void* buf = taosMemoryCalloc(1, len); - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, len); - tEncodeMqDataRsp(&encoder, &rspObj->rsp); - tEncoderClear(&encoder); - - raw->raw = buf; - raw->raw_len = len; raw->raw_type = RES_TYPE__TMQ; uDebug("tmq get raw type data:%p", raw); } else if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res); - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeSTaosxRsp, &rspObj->rsp, len, code); - if (code < 0) { - return -1; + if(encodeMqDataRsp(tEncodeSTaosxRsp, &rspObj->rsp, raw) != 0){ + uError("tmq get raw type error:%d", terrno); + return terrno; } - - void* buf = taosMemoryCalloc(1, len); - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, len); - tEncodeSTaosxRsp(&encoder, &rspObj->rsp); - tEncoderClear(&encoder); - - raw->raw = buf; - raw->raw_len = len; raw->raw_type = RES_TYPE__TMQ_METADATA; uDebug("tmq get raw type metadata:%p", raw); } else { @@ -2010,6 +2081,7 @@ void tmq_free_raw(tmq_raw_data raw) { if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) { taosMemoryFree(raw.raw); } + memset(terrMsg, 0, ERR_MSG_LEN); } int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 4cac1febe5..db86561e9b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -622,9 +622,9 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c if (TD_RES_TMQ(pRes)) { SMqRspObj* pRspObj = (SMqRspObj*)pRes; - pTopicName = pRspObj->topic; - vgId = pRspObj->vgId; - offsetVal = pRspObj->rsp.rspOffset; + pTopicName = pRspObj->common.topic; + vgId = pRspObj->common.vgId; + offsetVal = pRspObj->rsp.common.rspOffset; } else if (TD_RES_TMQ_META(pRes)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; pTopicName = pMetaRspObj->topic; @@ -632,9 +632,9 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c offsetVal = pMetaRspObj->metaRsp.rspOffset; } else if (TD_RES_TMQ_METADATA(pRes)) { SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; - pTopicName = pRspObj->topic; - vgId = pRspObj->vgId; - offsetVal = pRspObj->rsp.rspOffset; + pTopicName = pRspObj->common.topic; + vgId = pRspObj->common.vgId; + offsetVal = pRspObj->rsp.common.rspOffset; } else { code = TSDB_CODE_TMQ_INVALID_MSG; goto end; @@ -1056,17 +1056,17 @@ static void tmqMgmtInit(void) { } } -#define SET_ERROR_MSG(MSG) \ +#define SET_ERROR_MSG_TMQ(MSG) \ if (errstr != NULL) snprintf(errstr, errstrLen, MSG); tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (conf == NULL) { - SET_ERROR_MSG("configure is null") + SET_ERROR_MSG_TMQ("configure is null") return NULL; } taosThreadOnce(&tmqInit, tmqMgmtInit); if (tmqInitRes != 0) { terrno = tmqInitRes; - SET_ERROR_MSG("tmq timer init error") + SET_ERROR_MSG_TMQ("tmq timer init error") return NULL; } @@ -1074,7 +1074,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("failed to create consumer, groupId:%s, code:%s", conf->groupId, terrstr()); - SET_ERROR_MSG("malloc tmq failed") + SET_ERROR_MSG_TMQ("malloc tmq failed") return NULL; } @@ -1090,7 +1090,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); - SET_ERROR_MSG("malloc tmq element failed or group is empty") + SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty") goto _failed; } @@ -1123,7 +1123,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (tsem_init(&pTmq->rspSem, 0, 0) != 0) { tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId); - SET_ERROR_MSG("init t_sem failed") + SET_ERROR_MSG_TMQ("init t_sem failed") goto _failed; } @@ -1132,13 +1132,13 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (pTmq->pTscObj == NULL) { tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); tsem_destroy(&pTmq->rspSem); - SET_ERROR_MSG("init tscObj failed") + SET_ERROR_MSG_TMQ("init tscObj failed") goto _failed; } pTmq->refId = taosAddRef(tmqMgmt.rsetId, pTmq); if (pTmq->refId < 0) { - SET_ERROR_MSG("add tscObj ref failed") + SET_ERROR_MSG_TMQ("add tscObj ref failed") goto _failed; } @@ -1377,7 +1377,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if(tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp, *(int8_t*)POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead))) < 0){ + if(tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0){ tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; @@ -1387,9 +1387,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.rspOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset); tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, - tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId); + tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId); } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); @@ -1404,7 +1404,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if(tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp, *(int8_t*)POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead))) < 0){ + if(tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0){ tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; @@ -1598,6 +1598,9 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); + if(pRspObj == NULL) { + return NULL; + } pRspObj->resType = RES_TYPE__TMQ_META; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); @@ -1611,8 +1614,7 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { void changeByteEndian(char* pData){ char* p = pData; - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | - // version: + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | version: int32_t blockVersion = *(int32_t*)p; ASSERT(blockVersion == BLOCK_VERSION_1); *(int32_t*)p = BLOCK_VERSION_2; @@ -1649,7 +1651,7 @@ static void tmqGetRawDataRowsPrecisionFromRes(void *pRetrieve, void** rawData, i } } -static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) { +static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) { (*numOfRows) = 0; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); @@ -1660,14 +1662,14 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - bool needTransformSchema = !pRspObj->rsp.withSchema; - if (!pRspObj->rsp.withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable - pRspObj->rsp.withSchema = true; - pRspObj->rsp.blockSchema = taosArrayInit(pRspObj->rsp.blockNum, sizeof(void*)); + bool needTransformSchema = !pDataRsp->withSchema; + if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable + pDataRsp->withSchema = true; + pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*)); } // extract the rows in this data packet - for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i); + for (int32_t i = 0; i < pDataRsp->blockNum; ++i) { + void* pRetrieve = taosArrayGetP(pDataRsp->blockData, i); void* rawData = NULL; int64_t rows = 0; // deal with compatibility @@ -1679,7 +1681,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); if(schema){ - taosArrayPush(pRspObj->rsp.blockSchema, &schema); + taosArrayPush(pDataRsp->blockSchema, &schema); } } } @@ -1687,18 +1689,24 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); - pRspObj->resType = RES_TYPE__TMQ; + if(pRspObj == NULL){ + return NULL; + } + pRspObj->common.resType = RES_TYPE__TMQ; memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); - tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj); + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); return pRspObj; } SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); - pRspObj->resType = RES_TYPE__TMQ_METADATA; + if(pRspObj == NULL){ + return NULL; + } + pRspObj->common.resType = RES_TYPE__TMQ_METADATA; memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp)); - tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj); + tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common); return pRspObj; } @@ -1891,7 +1899,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; + SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->dataRsp; if (pDataRsp->head.epoch == consumerEpoch) { taosWLockLatch(&tmq->lock); @@ -1994,8 +2002,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp; - if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { + if (pDataRsp->head.epoch == consumerEpoch) { taosWLockLatch(&tmq->lock); SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); pollRspWrapper->vgHandle = pVg; @@ -2009,11 +2018,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { return NULL; } - updateVgInfo(pVg, &pollRspWrapper->taosxRsp.reqOffset, &pollRspWrapper->taosxRsp.rspOffset, - pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId, - pollRspWrapper->taosxRsp.blockNum != 0); + updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, + pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId, + pDataRsp->blockNum != 0); - if (pollRspWrapper->taosxRsp.blockNum == 0) { + if (pDataRsp->blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId); pVg->emptyBlockReceiveTs = taosGetTimestampMs(); @@ -2034,7 +2043,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, + tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); taosFreeQitem(pRspWrapper); @@ -2042,7 +2051,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { return pRsp; } else { tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); + tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); @@ -2182,7 +2191,12 @@ const char* tmq_err2str(int32_t err) { } else if (err == -1) { return "fail"; } else { - return tstrerror(err); + if(*(taosGetErrMsg()) == 0){ + return tstrerror(err); + } else{ + snprintf(taosGetErrMsgReturn(), ERR_MSG_LEN, "%s,detail:%s", tstrerror(err), taosGetErrMsg()); + return (const char*)taosGetErrMsgReturn(); + } } } @@ -2205,15 +2219,11 @@ const char* tmq_get_topic_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - return strchr(pRspObj->topic, '.') + 1; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + return strchr(((SMqRspObjCommon*)res)->topic, '.') + 1; } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return strchr(pMetaRspObj->topic, '.') + 1; - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - return strchr(pRspObj->topic, '.') + 1; } else { return NULL; } @@ -2224,15 +2234,11 @@ const char* tmq_get_db_name(TAOS_RES* res) { return NULL; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - return strchr(pRspObj->db, '.') + 1; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + return strchr(((SMqRspObjCommon*)res)->db, '.') + 1; } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return strchr(pMetaRspObj->db, '.') + 1; - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - return strchr(pRspObj->db, '.') + 1; } else { return NULL; } @@ -2242,15 +2248,11 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (res == NULL) { return -1; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - return pRspObj->vgId; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + return ((SMqRspObjCommon*)res)->vgId; } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return pMetaRspObj->vgId; - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - return pRspObj->vgId; } else { return -1; } @@ -2260,24 +2262,19 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { if (res == NULL) { return TSDB_CODE_INVALID_PARA; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - STqOffsetVal* pOffset = &pRspObj->rsp.reqOffset; - if (pOffset->type == TMQ_OFFSET__LOG) { - return pRspObj->rsp.reqOffset.version; + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); + STqOffsetVal* pOffset = &common->reqOffset; + if (common->reqOffset.type == TMQ_OFFSET__LOG) { + return common->reqOffset.version; } else { - tscError("invalid offset type:%d", pOffset->type); + tscError("invalid offset type:%d", common->reqOffset.type); } } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pRspObj = (SMqMetaRspObj*)res; if (pRspObj->metaRsp.rspOffset.type == TMQ_OFFSET__LOG) { return pRspObj->metaRsp.rspOffset.version; } - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - if (pRspObj->rsp.reqOffset.type == TMQ_OFFSET__LOG) { - return pRspObj->rsp.reqOffset.version; - } } else { tscError("invalid tmq type:%d", *(int8_t*)res); } @@ -2290,20 +2287,15 @@ const char* tmq_get_table_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 || - pRspObj->resIter >= pRspObj->rsp.blockNum) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); + + SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; + if (!common->withTbName || common->blockTbName == NULL || pRspObj->resIter < 0 || + pRspObj->resIter >= common->blockNum) { return NULL; } - return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); - } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; - if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 || - pRspObj->resIter >= pRspObj->rsp.blockNum) { - return NULL; - } - return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); + return (const char*)taosArrayGetP(common->blockTbName, pRspObj->resIter); } return NULL; } @@ -2640,17 +2632,17 @@ void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, cons } SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { - SMqRspObj* pRspObj = (SMqRspObj*)res; + SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); + SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; pRspObj->resIter++; - - if (pRspObj->resIter < pRspObj->rsp.blockNum) { - if (pRspObj->rsp.withSchema) { + if (pRspObj->resIter < common->blockNum) { + if (common->withSchema) { doFreeReqResultInfo(&pRspObj->resInfo); - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter); setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); } - void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); + void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter); void* rawData = NULL; int64_t rows = 0; int32_t precision = 0; @@ -2685,14 +2677,14 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { SMqDataRsp rsp; SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - tDecodeMqDataRsp(&decoder, &rsp, *(int8_t*)POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead))); + tDecodeMqDataRsp(&decoder, &rsp); tDecoderClear(&decoder); SMqRspHead* pHead = pMsg->pData; tmq_topic_assignment assignment = {.begin = pHead->walsver, .end = pHead->walever + 1, - .currentOffset = rsp.rspOffset.version, + .currentOffset = rsp.common.rspOffset.version, .vgId = pParam->vgId}; taosThreadMutexLock(&pCommon->mutex); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7db408f028..340463e48c 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9173,7 +9173,7 @@ int32_t tDecodeMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) { void tDeleteMqMetaRsp(SMqMetaRsp *pRsp) { taosMemoryFree(pRsp->metaRsp); } -int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) { +int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRspCommon *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; @@ -9198,14 +9198,13 @@ int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) { return 0; } -int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { - if (tEncodeI8(pEncoder, MQ_DATA_RSP_VERSION) < 0) return -1; +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const void *pRsp) { if (tEncodeMqDataRspCommon(pEncoder, pRsp) < 0) return -1; - if (tEncodeI64(pEncoder, pRsp->sleepTime) < 0) return -1; + if (tEncodeI64(pEncoder, ((SMqDataRsp*)pRsp)->sleepTime) < 0) return -1; return 0; } -int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { +int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRspCommon *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; @@ -9251,19 +9250,17 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { return 0; } -int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp, int8_t dataVersion) { - if (dataVersion >= MQ_DATA_RSP_VERSION) { - if (tDecodeI8(pDecoder, &dataVersion) < 0) return -1; - } +int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) { if (tDecodeMqDataRspCommon(pDecoder, pRsp) < 0) return -1; if (!tDecodeIsEnd(pDecoder)) { - if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1; + if (tDecodeI64(pDecoder, &((SMqDataRsp*)pRsp)->sleepTime) < 0) return -1; } return 0; } -void tDeleteMqDataRsp(SMqDataRsp *pRsp) { +static void tDeleteMqDataRspCommon(void *rsp) { + SMqDataRspCommon *pRsp = rsp; pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; @@ -9275,10 +9272,14 @@ void tDeleteMqDataRsp(SMqDataRsp *pRsp) { tOffsetDestroy(&pRsp->rspOffset); } -int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { - if (tEncodeI8(pEncoder, MQ_DATA_RSP_VERSION) < 0) return -1; - if (tEncodeMqDataRspCommon(pEncoder, (const SMqDataRsp *)pRsp) < 0) return -1; +void tDeleteMqDataRsp(void *rsp) { + tDeleteMqDataRspCommon(rsp); +} +int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const void *rsp) { + if (tEncodeMqDataRspCommon(pEncoder, rsp) < 0) return -1; + + const STaosxRsp *pRsp = (const STaosxRsp *)rsp; if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { for (int32_t i = 0; i < pRsp->createTableNum; i++) { @@ -9290,19 +9291,17 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { return 0; } -int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp, int8_t dataVersion) { - if (dataVersion >= MQ_DATA_RSP_VERSION) { - if (tDecodeI8(pDecoder, &dataVersion) < 0) return -1; - } - if (tDecodeMqDataRspCommon(pDecoder, (SMqDataRsp *)pRsp) < 0) return -1; +int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, void *rsp) { + if (tDecodeMqDataRspCommon(pDecoder, rsp) < 0) return -1; + STaosxRsp *pRsp = (STaosxRsp *)rsp; if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *)); for (int32_t i = 0; i < pRsp->createTableNum; i++) { - void *pCreate = NULL; - uint64_t len; + void * pCreate = NULL; + uint64_t len = 0; if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1; int32_t l = (int32_t)len; taosArrayPush(pRsp->createTableLen, &l); @@ -9313,20 +9312,13 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp, int8_t dataVersion return 0; } -void tDeleteSTaosxRsp(STaosxRsp *pRsp) { - pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); - taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); - pRsp->blockData = NULL; - taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); - pRsp->blockSchema = NULL; - taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree); - pRsp->blockTbName = NULL; +void tDeleteSTaosxRsp(void *rsp) { + tDeleteMqDataRspCommon(rsp); + STaosxRsp *pRsp = (STaosxRsp *)rsp; pRsp->createTableLen = taosArrayDestroy(pRsp->createTableLen); taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); pRsp->createTableReq = NULL; - tOffsetDestroy(&pRsp->reqOffset); - tOffsetDestroy(&pRsp->rspOffset); } int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index ed9333f480..c7ae941389 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -648,7 +648,6 @@ static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscri _over: mndReleaseConsumer(pMnode, pExistedConsumer); tDeleteSMqConsumerObj(pConsumerNew); - taosArrayDestroyP(subscribe->topicNames, (FDelete)taosMemoryFree); return NULL; } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index f17716eda0..b717504e1e 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -58,17 +58,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer streamSetupScheduleTrigger(pTask); SCheckpointInfo *pChkInfo = &pTask->chkInfo; - - // checkpoint ver is the kept version, handled data should be the next version. - if (pChkInfo->checkpointId != 0) { - pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1; - pChkInfo->processedVer = pChkInfo->checkpointVer; - pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer; - pTask->execInfo.startCheckpointId = pChkInfo->checkpointId; - - sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, - pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); - } + tqSetRestoreVersionInfo(pTask); char* p = streamTaskGetStatus(pTask)->name; if (pTask->info.fillHistory) { diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 2a076cfc61..bd8b73ed33 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -119,8 +119,8 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t // tqExec int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded); -int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, +int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp, int32_t type, int32_t vgId); int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); @@ -154,9 +154,9 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tq util int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); -int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset); +int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset); void tqUpdateNodeStage(STQ* pTq, bool isLeader); int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, SSubmitTbData* pTableData, const char* id); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8edc0fed4d..567d61e27a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -153,10 +153,10 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { } SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, req.reqOffset); - dataRsp.blockNum = 0; + tqInitDataRsp(&dataRsp.common, req.reqOffset); + dataRsp.common.blockNum = 0; char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.reqOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset); tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, reqId:0x%" PRIx64, req.consumerId, vgId, buf, req.reqId); @@ -165,7 +165,7 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { return 0; } -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp, int32_t type, int32_t vgId) { int64_t sver = 0, ever = 0; walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); @@ -174,11 +174,11 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* char buf1[TSDB_OFFSET_LEN] = {0}; char buf2[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf1, TSDB_OFFSET_LEN, &pRsp->reqOffset); - tFormatOffset(buf2, TSDB_OFFSET_LEN, &pRsp->rspOffset); + tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset); + tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset); tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, - vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); + vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId); return 0; } @@ -499,7 +499,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { taosRUnLockLatch(&pTq->lock); SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, req.reqOffset); + tqInitDataRsp(&dataRsp.common, req.reqOffset); if (req.useSnapshot == true) { tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey); @@ -508,10 +508,10 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } - dataRsp.rspOffset.type = TMQ_OFFSET__LOG; + dataRsp.common.rspOffset.type = TMQ_OFFSET__LOG; if (reqOffset.type == TMQ_OFFSET__LOG) { - dataRsp.rspOffset.version = reqOffset.version; + dataRsp.common.rspOffset.version = reqOffset.version; } else if (reqOffset.type < 0) { STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); if (pOffset != NULL) { @@ -522,17 +522,17 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } - dataRsp.rspOffset.version = pOffset->val.version; + dataRsp.common.rspOffset.version = pOffset->val.version; tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from store:%" PRId64, consumerId, vgId, - req.subKey, dataRsp.rspOffset.version); + req.subKey, dataRsp.common.rspOffset.version); } else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) { - dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position + dataRsp.common.rspOffset.version = sver; // not consume yet, set the earliest position } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - dataRsp.rspOffset.version = ever; + dataRsp.common.rspOffset.version = ever; } tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from init:%" PRId64, consumerId, vgId, req.subKey, - dataRsp.rspOffset.version); + dataRsp.common.rspOffset.version); } } else { tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey, @@ -760,16 +760,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { streamSetupScheduleTrigger(pTask); SCheckpointInfo* pChkInfo = &pTask->chkInfo; - - // checkpoint ver is the kept version, handled data should be the next version. - if (pChkInfo->checkpointId != 0) { - pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1; - pChkInfo->processedVer = pChkInfo->checkpointVer; - pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer; - pTask->execInfo.startCheckpointId = pChkInfo->checkpointId; - tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr, - pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); - } + tqSetRestoreVersionInfo(pTask); char* p = streamTaskGetStatus(pTask)->name; const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 017d5247d8..08f1689f2f 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -15,7 +15,7 @@ #include "tq.h" -int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { +int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision) { int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { @@ -30,22 +30,22 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRspForTmq); - taosArrayPush(pRsp->blockDataLen, &actualLen); - taosArrayPush(pRsp->blockData, &buf); + taosArrayPush(((SMqDataRspCommon*)pRsp)->blockDataLen, &actualLen); + taosArrayPush(((SMqDataRspCommon*)pRsp)->blockData, &buf); return TSDB_CODE_SUCCESS; } -static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) { +static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, void* pRsp) { SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper); if (pSW == NULL) { return -1; } - taosArrayPush(pRsp->blockSchema, &pSW); + taosArrayPush(((SMqDataRspCommon*)pRsp)->blockSchema, &pSW); return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, void* pRsp, int32_t n) { SMetaReader mr = {0}; metaReaderDoInit(&mr, pTq->pVnode->pMeta, META_READER_LOCK); @@ -57,7 +57,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in for (int32_t i = 0; i < n; i++) { char* tbName = taosStrdup(mr.me.name); - taosArrayPush(pRsp->blockTbName, &tbName); + taosArrayPush(((SMqDataRspCommon*)pRsp)->blockTbName, &tbName); } metaReaderClear(&mr); return 0; @@ -125,7 +125,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* return code; } - pRsp->blockNum++; + pRsp->common.blockNum++; if (pDataBlock == NULL) { blockDataDestroy(pHandle->block); pHandle->block = NULL; @@ -149,7 +149,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* return code; } - pRsp->blockNum++; + pRsp->common.blockNum++; totalRows += pDataBlock->info.rows; if (totalRows >= tmqRowSize) { break; @@ -158,8 +158,8 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* } tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", - pHandle->consumerId, vgId, pRsp->blockNum, totalRows); - qStreamExtractOffset(task, &pRsp->rspOffset); + pHandle->consumerId, vgId, pRsp->common.blockNum, totalRows); + qStreamExtractOffset(task, &pRsp->common.rspOffset); return 0; } @@ -186,7 +186,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta tqDebug("tmqsnap task execute end, get %p", pDataBlock); if (pDataBlock != NULL && pDataBlock->info.rows > 0) { - if (pRsp->withTbName) { + if (pRsp->common.withTbName) { if (pOffset->type == TMQ_OFFSET__LOG) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) { @@ -194,21 +194,21 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } } else { char* tbName = taosStrdup(qExtractTbnameFromTask(task)); - taosArrayPush(pRsp->blockTbName, &tbName); + taosArrayPush(pRsp->common.blockTbName, &tbName); } } - if (pRsp->withSchema) { + if (pRsp->common.withSchema) { if (pOffset->type == TMQ_OFFSET__LOG) { tqAddBlockSchemaToRsp(pExec, pRsp); } else { SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); - taosArrayPush(pRsp->blockSchema, &pSW); + taosArrayPush(pRsp->common.blockSchema, &pSW); } } tqAddBlockDataToRsp(pDataBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pDataBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision); - pRsp->blockNum++; + pRsp->common.blockNum++; if (pOffset->type == TMQ_OFFSET__LOG) { continue; } else { @@ -234,13 +234,13 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); - qStreamExtractOffset(task, &pRsp->rspOffset); + qStreamExtractOffset(task, &pRsp->common.rspOffset); break; } - if (pRsp->blockNum > 0) { + if (pRsp->common.blockNum > 0) { tqDebug("tmqsnap task exec exited, get data"); - qStreamExtractOffset(task, &pRsp->rspOffset); + qStreamExtractOffset(task, &pRsp->common.rspOffset); break; } } @@ -268,7 +268,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto loop_table; } - if (pRsp->withTbName) { + if (pRsp->common.withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { goto loop_table; @@ -312,8 +312,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR *totalRows += pBlock->info.rows; blockDataFreeRes(pBlock); SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); - taosArrayPush(pRsp->blockSchema, &pSW); - pRsp->blockNum++; + taosArrayPush(pRsp->common.blockSchema, &pSW); + pRsp->common.blockNum++; } continue; loop_table: @@ -336,7 +336,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) { goto loop_db; } - if (pRsp->withTbName) { + if (pRsp->common.withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { goto loop_db; @@ -380,8 +380,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR *totalRows += pBlock->info.rows; blockDataFreeRes(pBlock); SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); - taosArrayPush(pRsp->blockSchema, &pSW); - pRsp->blockNum++; + taosArrayPush(pRsp->common.blockSchema, &pSW); + pRsp->common.blockNum++; } continue; loop_db: diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 8099d6a6f2..28f4a19949 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -18,7 +18,7 @@ static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); -int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { +int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { tOffsetCopy(&pRsp->reqOffset, &pOffset); tOffsetCopy(&pRsp->rspOffset, &pOffset); @@ -39,7 +39,7 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { streamMetaUpdateStageRole(pTq->pStreamMeta, state.term, isLeader); } -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { +static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { tOffsetCopy(&pRsp->reqOffset, &pOffset); tOffsetCopy(&pRsp->rspOffset, &pOffset); @@ -110,9 +110,9 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand SMqDataRsp dataRsp = {0}; tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1); - tqInitDataRsp(&dataRsp, *pOffsetVal); + tqInitDataRsp(&dataRsp.common, *pOffsetVal); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.rspOffset.version); + pHandle->subKey, vgId, dataRsp.common.rspOffset.version); int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); tDeleteMqDataRsp(&dataRsp); @@ -137,7 +137,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, terrno = 0; SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, *pOffset); + tqInitDataRsp(&dataRsp.common, *pOffset); qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); int code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest); @@ -146,11 +146,11 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) { + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.common.blockNum == 0) { // lock taosWLockLatch(&pTq->lock); int64_t ver = walGetCommittedVer(pTq->pVnode->pWal); - if (dataRsp.rspOffset.version > ver) { // check if there are data again to avoid lost data + if (dataRsp.common.rspOffset.version > ver) { // check if there are data again to avoid lost data code = tqRegisterPushHandle(pTq, pHandle, pMsg); taosWUnLockLatch(&pTq->lock); goto end; @@ -158,15 +158,15 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, taosWUnLockLatch(&pTq->lock); } - tOffsetCopy(&dataRsp.reqOffset, pOffset); // reqOffset represents the current date offset, may be changed if wal not exists - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + tOffsetCopy(&dataRsp.common.reqOffset, pOffset); // reqOffset represents the current date offset, may be changed if wal not exists + code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.rspOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d", - consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); + consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code); tDeleteMqDataRsp(&dataRsp); return code; } @@ -194,7 +194,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, int32_t vgId = TD_VID(pTq->pVnode); SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, *offset); + tqInitTaosxRsp(&taosxRsp.common, *offset); if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { @@ -214,13 +214,13 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, - taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts); - if (taosxRsp.blockNum > 0) { - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.common.blockNum, taosxRsp.common.rspOffset.type, + taosxRsp.common.rspOffset.uid, taosxRsp.common.rspOffset.ts); + if (taosxRsp.common.blockNum > 0) { + code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } else { - tOffsetCopy(offset, &taosxRsp.rspOffset); + tOffsetCopy(offset, &taosxRsp.common.rspOffset); } } @@ -235,9 +235,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, ASSERT(savedEpoch <= pRequest->epoch); if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer); code = tqSendDataRsp( - pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, + pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } @@ -249,9 +249,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, // process meta if (pHead->msgType != TDMT_VND_SUBMIT) { if (totalRows > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer); code = tqSendDataRsp( - pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, + pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } @@ -292,9 +292,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); + tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer + 1); code = tqSendDataRsp( - pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, + pHandle, pMsg, pRequest, &taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } else { @@ -386,7 +386,7 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll return 0; } -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever) { int32_t len = 0; int32_t code = 0; @@ -394,7 +394,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); + tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code); } if (code < 0) { @@ -418,7 +418,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { tEncodeMqDataRsp(&encoder, pRsp); } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); + tEncodeSTaosxRsp(&encoder, pRsp); } tEncoderClear(&encoder); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index d2c7924cf5..4ce8579ea0 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -86,6 +86,22 @@ int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode) return TSDB_CODE_SUCCESS; } +void tqSetRestoreVersionInfo(SStreamTask* pTask) { + SCheckpointInfo* pChkInfo = &pTask->chkInfo; + + // checkpoint ver is the kept version, handled data should be the next version. + if (pChkInfo->checkpointId != 0) { + pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1; + pChkInfo->processedVer = pChkInfo->checkpointVer; + pTask->execInfo.startCheckpointId = pChkInfo->checkpointId; + + tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr, + pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); + } + + pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer; +} + int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) { int32_t vgId = pMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); diff --git a/source/libs/audit/src/auditMain.c b/source/libs/audit/src/auditMain.c index 19dc771c56..96934888eb 100644 --- a/source/libs/audit/src/auditMain.c +++ b/source/libs/audit/src/auditMain.c @@ -27,7 +27,7 @@ #include "osMemory.h" SAudit tsAudit = {0}; -char* tsAuditUri = "/audit"; +char* tsAuditUri = "/audit_v2"; char* tsAuditBatchUri = "/audit-batch"; int32_t auditInit(const SAuditCfg *pCfg) { diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 23e873d335..115a61f647 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -108,6 +108,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe goto _error; } + // todd: the pk information should comes from the physical plan for(int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { SColMatchItem* pItem = taosArrayGet(pInfo->matchInfo.pList, i); if (pItem->isPk) { @@ -223,8 +224,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pBufferedRes); taosArrayClear(pInfo->pUidList); - int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferedRes, pInfo->pSlotIds, - pInfo->pDstSlotIds, pInfo->pUidList); + int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferedRes, + pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -293,9 +294,11 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } if (NULL == pInfo->pLastrowReader) { - code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, - taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader, - pTaskInfo->id.str, pInfo->pFuncTypeList, &pInfo->pkCol, pInfo->numOfPks); + code = pInfo->readHandle.api.cacheFn.openReader( + pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, taosArrayGetSize(pInfo->matchInfo.pList), + pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader, pTaskInfo->id.str, pInfo->pFuncTypeList, + &pInfo->pkCol, pInfo->numOfPks); + if (code != TSDB_CODE_SUCCESS) { pInfo->currentGroupIndex += 1; taosArrayClear(pInfo->pUidList); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 7e99867b2a..6cd10f8a1f 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -641,7 +641,7 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) { } int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* tFields, - int numFields, bool needChangeLength) { + int numFields, bool needChangeLength, char* errstr, int32_t errstrLen) { void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; @@ -662,8 +662,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } char* p = (char*)data; - // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each - // column length | + // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each column length | int32_t version = *(int32_t*)data; p += sizeof(int32_t); p += sizeof(int32_t); @@ -689,12 +688,12 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; if (tFields != NULL && numFields != numOfCols) { - uError("numFields:%d != numOfCols:%d", numFields, numOfCols); + if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d != raw numOfCols:%d", numFields, numOfCols); ret = TSDB_CODE_INVALID_PARA; goto end; } if (tFields != NULL && numFields > boundInfo->numOfBound) { - uError("numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound); + if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound); ret = TSDB_CODE_INVALID_PARA; goto end; } @@ -703,7 +702,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SSchema* pColSchema = &pSchema[j]; SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { - uError("type or bytes not equal"); + if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d", + pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t))); ret = TSDB_CODE_INVALID_PARA; goto end; } @@ -732,7 +732,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SSchema* pColSchema = &pSchema[j]; if (strcmp(pColSchema->name, tFields[i].name) == 0) { if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { - uError("type or bytes not equal"); + if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d", + pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t))); ret = TSDB_CODE_INVALID_PARA; goto end; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e7e0be6d57..a824979801 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -570,13 +570,14 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu if (pScanLogicNode->pVgroupList) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } - int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); + int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); if (TSDB_CODE_SUCCESS == code && pScanLogicNode->pFuncTypes != NULL) { pScan->pFuncTypes = taosArrayInit(taosArrayGetSize(pScanLogicNode->pFuncTypes), sizeof(int32_t)); if (NULL == pScan->pFuncTypes) { return TSDB_CODE_OUT_OF_MEMORY; } + SNode* pTargetNode = NULL; int funcTypeIndex = 0; FOREACH(pTargetNode, ((SScanPhysiNode*)pScan)->pScanCols) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b4c06e135a..f7f790fbe7 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1288,7 +1288,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { } SStreamTask* pTask = *(SStreamTask**)pIter; - stDebug("vgId:%d s-task:%s set closing flag", vgId, pTask->id.idStr); + stDebug("vgId:%d s-task:%s set task closing flag", vgId, pTask->id.idStr); streamTaskStop(pTask); } @@ -1649,6 +1649,13 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 return 0; } + void* p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if (p == NULL) { // task does not exists in current vnode, not record the complete info + qError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId); + streamMetaWUnLock(pMeta); + return 0; + } + SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet; STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready}; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 70c2619f6f..7dc93ceccf 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -993,16 +993,17 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t SDownstreamStatusInfo* p = taosArrayGet(pInfo->pList, i); if (p->taskId == taskId) { ASSERT(reqId == p->reqId); - p->status = status; - p->rspTs = rspTs; // count down one, since it is ready now - if (p->status == TASK_DOWNSTREAM_READY) { + if ((p->status != TASK_DOWNSTREAM_READY) && (status == TASK_DOWNSTREAM_READY)) { *pNotReady = atomic_sub_fetch_32(&pInfo->notReadyTasks, 1); } else { *pNotReady = pInfo->notReadyTasks; } + p->status = status; + p->rspTs = rspTs; + taosThreadMutexUnlock(&pInfo->checkInfoLock); return TSDB_CODE_SUCCESS; } @@ -1026,7 +1027,7 @@ int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) { } taosThreadMutexUnlock(&pInfo->checkInfoLock); - stDebug("s-task:%s set the in check procedure flag", id); + stDebug("s-task:%s set the in-check-procedure flag", id); return 0; } @@ -1038,7 +1039,7 @@ int32_t streamTaskCompleteCheck(STaskCheckInfo* pInfo, const char* id) { } int64_t el = taosGetTimestampMs() - pInfo->startTs; - stDebug("s-task:%s check downstream completed, elapsed time:%" PRId64 " ms", id, el); + stDebug("s-task:%s clear the in-check-procedure flag, elapsed time:%" PRId64 " ms", id, el); pInfo->startTs = 0; pInfo->inCheckProcess = 0; @@ -1098,27 +1099,36 @@ static void rspMonitorFn(void* param, void* tmrId) { int64_t now = taosGetTimestampMs(); int64_t el = now - pInfo->startTs; ETaskStatus state = pStat->state; - int32_t numOfReady = 0; int32_t numOfFault = 0; + const char* id = pTask->id.idStr; - stDebug("s-task:%s start to do check downstream rsp check", pTask->id.idStr); + stDebug("s-task:%s start to do check downstream rsp check", id); - if (state == TASK_STATUS__STOP || state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) { + if (state == TASK_STATUS__STOP) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s vgId:%d quit from monitor rsp tmr, ref:%d", pTask->id.idStr, pStat->name, vgId, ref); - streamTaskCompleteCheck(pInfo, pTask->id.idStr); + stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref); + streamTaskCompleteCheck(pInfo, id); + + streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false); + return; + } + + if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref); + streamTaskCompleteCheck(pInfo, id); return; } taosThreadMutexLock(&pInfo->checkInfoLock); if (pInfo->notReadyTasks == 0) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", pTask->id.idStr, - pStat->name, vgId, ref); + stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat->name, + vgId, ref); taosThreadMutexUnlock(&pInfo->checkInfoLock); - streamTaskCompleteCheck(pInfo, pTask->id.idStr); + streamTaskCompleteCheck(pInfo, id); return; } @@ -1131,13 +1141,12 @@ static void rspMonitorFn(void* param, void* tmrId) { if (p->status == TASK_DOWNSTREAM_READY) { numOfReady += 1; } else if (p->status == TASK_UPSTREAM_NEW_STAGE || p->status == TASK_DOWNSTREAM_NOT_LEADER) { - stDebug("s-task:%s recv status from downstream, task:0x%x, quit from check downstream tasks", pTask->id.idStr, - p->taskId); + stDebug("s-task:%s recv status from downstream, task:0x%x, quit from check downstream tasks", id, p->taskId); numOfFault += 1; - } else { // TASK_DOWNSTREAM_NOT_READY - if (p->rspTs == 0) { // not response yet + } else { // TASK_DOWNSTREAM_NOT_READY + if (p->rspTs == 0) { // not response yet ASSERT(p->status == -1); - if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. + if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. taosArrayPush(pTimeoutList, &p->taskId); } else { // el < CHECK_NOT_RSP_DURATION // do nothing and continue waiting for their rsps @@ -1148,25 +1157,26 @@ static void rspMonitorFn(void* param, void* tmrId) { } } } else { // unexpected status - stError("s-task:%s unexpected task status:%s during waiting for check rsp", pTask->id.idStr, pStat->name); + stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat->name); } int32_t numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList); int32_t numOfTimeout = (int32_t)taosArrayGetSize(pTimeoutList); // fault tasks detected, not try anymore - if (((numOfReady + numOfFault + numOfNotReady + numOfTimeout) == taosArrayGetSize(pInfo->pList)) && (numOfFault > 0)) { + if (((numOfReady + numOfFault + numOfNotReady + numOfTimeout) == taosArrayGetSize(pInfo->pList)) && + (numOfFault > 0)) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug( "s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart " - "detected, ref:%d", - pTask->id.idStr, pStat->name, vgId, ref); + "detected, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d", + id, pStat->name, vgId, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); taosThreadMutexUnlock(&pInfo->checkInfoLock); taosArrayDestroy(pNotReadyList); taosArrayDestroy(pTimeoutList); - streamTaskCompleteCheck(pInfo, pTask->id.idStr); + streamTaskCompleteCheck(pInfo, id); return; } @@ -1176,14 +1186,14 @@ static void rspMonitorFn(void* param, void* tmrId) { stDebug( "s-task:%s status:%s vgId:%d stopped by other threads to check downstream process, notReady:%d, fault:%d, " "timeout:%d, ready:%d ref:%d", - pTask->id.idStr, pStat->name, vgId, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); + id, pStat->name, vgId, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); taosThreadMutexUnlock(&pInfo->checkInfoLock); // add the not-ready tasks into the final task status result buf, along with related fill-history task if exists. streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false); if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo; - streamMetaAddTaskLaunchResult(pTask->pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId, pInfo->startTs, now, false); + STaskId* pHId = &pTask->hTaskInfo.id; + streamMetaAddTaskLaunchResult(pTask->pMeta, pHId->streamId, pHId->taskId, pInfo->startTs, now, false); } return; } @@ -1205,7 +1215,7 @@ static void rspMonitorFn(void* param, void* tmrId) { } } - stDebug("s-task:%s %d downstream task(s) not ready, send check msg again", pTask->id.idStr, numOfNotReady); + stDebug("s-task:%s %d downstream task(s) not ready, send check msg again", id, numOfNotReady); } if (numOfTimeout > 0) { @@ -1225,15 +1235,14 @@ static void rspMonitorFn(void* param, void* tmrId) { } } - stDebug("s-task:%s %d downstream tasks timeout, send check msg again, start ts:%" PRId64, pTask->id.idStr, - numOfTimeout, now); + stDebug("s-task:%s %d downstream tasks timeout, send check msg again, start ts:%" PRId64, id, numOfTimeout, now); } taosThreadMutexUnlock(&pInfo->checkInfoLock); taosTmrReset(rspMonitorFn, CHECK_RSP_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr); - stDebug("s-task:%s continue checking rsp in 200ms, notReady:%d, fault:%d, timeout:%d, ready:%d", pTask->id.idStr, - numOfNotReady, numOfFault, numOfTimeout, numOfReady); + stDebug("s-task:%s continue checking rsp in 200ms, notReady:%d, fault:%d, timeout:%d, ready:%d", id, numOfNotReady, + numOfFault, numOfTimeout, numOfReady); taosArrayDestroy(pNotReadyList); taosArrayDestroy(pTimeoutList); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ec81f57319..ac61b288a4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -27,8 +27,12 @@ typedef struct { } STaosError; static threadlocal int32_t tsErrno; +static threadlocal char tsErrMsgDetail[ERR_MSG_LEN] = {0}; +static threadlocal char tsErrMsgReturn[ERR_MSG_LEN] = {0}; int32_t* taosGetErrno() { return &tsErrno; } +char* taosGetErrMsg() { return tsErrMsgDetail; } +char* taosGetErrMsgReturn() { return tsErrMsgReturn; } #ifdef TAOS_ERROR_C #define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg)}, diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 29664759b7..a3459f6968 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -320,6 +320,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_delete.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 diff --git a/tests/pytest/util/csv.py b/tests/pytest/util/csv.py new file mode 100644 index 0000000000..6e1ee1732c --- /dev/null +++ b/tests/pytest/util/csv.py @@ -0,0 +1,62 @@ +import csv +import os + +class TDCsv: + def __init__(self): + self.__file_name = '' + self.__file_path = '' + + @property + def file_name(self): + return self.__file_name + + @file_name.setter + def file_name(self, value): + self.__file_name = value + + @property + def file_path(self): + return self.__file_path + + @file_path.setter + def file_path(self, value): + self.__file_path = value + + @property + def file(self): + if self.file_name and self.file_path: + return os.path.join(self.file_path, self.file_name) + return None + + + def read(self): + try: + with open(self.file, newline='') as csvfile: + reader = csv.reader(csvfile) + for row in reader: + print(row) + except Exception as errMsg: + raise Exception(errMsg) + + def __write_with_quotes(self, csv_writer, data): + for row in data: + row_with_quotes = [f'"{field}"' if isinstance(field, str) else field for field in row] + csv_writer.writerow(row_with_quotes) + + def write(self, data: dict): + try: + with open(self.file, 'w', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerows(data) + # self.__write_with_quotes(writer, data) + except Exception as errMsg: + raise Exception(errMsg) + + def delete(self): + try: + if os.path.exists(self.file): + os.remove(self.file) + except Exception as errMsg: + raise Exception(errMsg) + + diff --git a/tests/system-test/1-insert/composite_primary_key_create.py b/tests/system-test/1-insert/composite_primary_key_create.py new file mode 100644 index 0000000000..cde960739c --- /dev/null +++ b/tests/system-test/1-insert/composite_primary_key_create.py @@ -0,0 +1,224 @@ +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * + + +class IllegalDataType(Enum): + NULL = 'null' + BOOL = 'bool' + TINYINT = 'tinyint' + SMALLINT = 'smallint' + FLOAT = 'float' + DOUBLE = 'double' + TIMESTAMP = 'timestamp' + NCHAR = 'nchar(100)' + UTINYINT = 'tinyint unsigned' + USMALLINT = 'smallint unsigned' + JSON = 'json' + VARBINARY = 'varbinary(100)' + DECIMAL = 'decimal' + BLOB = 'blob' + MEDIUMBLOB = 'mediumblob' + GEOMETRY = 'geometry(512)' + EMPTY = '\'\'' + SPACE = '\' \'' + + +class LegalDataType(Enum): + INT = 'INT' + UINT = 'INT UNSIGNED' + BIGINT = 'BIGINT' + UBIGINT = 'BIGINT UNSIGNED' + VARCHAR = 'VARCHAR(100)' + BINARY = 'BINARY(100)' + +class LegalSpell(Enum): + CASE1 = 'primary key' + CASE2 = 'PRIMARY KEY' + CASE3 = 'Primary Key' + CASE4 = 'PriMary keY' + +class IllegalSpell(Enum): + CASE1 = 'primary' + CASE2 = 'key' + CASE3 = 'primay key' + CASE4 = 'primary ky' + CASE5 = 'primarykey' + CASE6 = 'key primary' + CASE7 = 'primay key primay key' + + +class TableType(Enum): + SUPERTABLE = 0 + CHILDTABLE = 1 + NORNALTABLE = 2 + +SHOW_LOG = True + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db_create_composite_primary_key" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") + tdSql.execute(f"use {self.database}") + + def check_pk_definition(self, table_name: str, d_type: LegalDataType, t_type: TableType): + tdSql.query(f"describe {table_name}", show=SHOW_LOG) + tdSql.checkData(1, 3, "PRIMARY KEY") + + if d_type == LegalDataType.BINARY: + d_type = LegalDataType.VARCHAR + + if t_type == TableType.SUPERTABLE: + expected_value = f"CREATE STABLE `{table_name}` (`ts` TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `pk` {d_type.value} ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium' PRIMARY KEY, `c2` INT ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium') TAGS (`engine` INT)" + elif t_type == TableType.NORNALTABLE: + expected_value = f"CREATE TABLE `{table_name}` (`ts` TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `pk` {d_type.value} ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium' PRIMARY KEY, `c2` INT ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium')" + + tdSql.query(f"show create table {table_name}", show=SHOW_LOG) + result = tdSql.queryResult + + tdSql.checkEqual("PRIMARY KEY" in result[0][1], True) + + def test_pk_datatype_legal(self, stable_name: str, ctable_name: str, ntable_name: str, dtype: LegalDataType): + # create super table and child table + tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG) + tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG) + + tdSql.execute(f"create table {stable_name} (ts timestamp, pk {dtype.value} primary key, c2 int) tags (engine int)", show=SHOW_LOG) + self.check_pk_definition(stable_name, dtype, TableType.SUPERTABLE) + + tdSql.execute(f"create table {ctable_name} using {stable_name} tags (0)", show=SHOW_LOG) + tdSql.execute(f"create table {ctable_name}_1 using {stable_name} tags (1) {ctable_name}_2 using {stable_name} tags (2) {ctable_name}_3 using {stable_name} tags (3)", show=SHOW_LOG) + + # create normal table + tdSql.execute(f"create table {ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 int)", show=SHOW_LOG) + self.check_pk_definition(ntable_name, dtype, TableType.NORNALTABLE) + + def test_pk_datatype_illegal(self, stable_name: str, ntable_name: str, dtype: LegalDataType): + # create super table and child table + tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG) + tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG) + + tdSql.error(f"create table {stable_name} (ts timestamp, pk {dtype.value} primary key, c2 int) tags (engine int)", show=SHOW_LOG) + + # create normal table + tdSql.error(f"create table {ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 int)", show=SHOW_LOG) + + def test_pk_spell_legal(self, stable_name: str, ntable_name: str, pk_spell: LegalSpell): + # create super table and child table + tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG) + tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG) + + tdSql.execute(f"create table {stable_name} (ts timestamp, pk int {pk_spell.value}, c2 int) tags (engine int)", show=SHOW_LOG) + + # create normal table + tdSql.execute(f"create table {ntable_name} (ts timestamp, pk int {pk_spell.value}, c2 int)", show=SHOW_LOG) + + def test_pk_spell_illegal(self, stable_name: str, ntable_name: str, pk_spell: LegalSpell): + # create super table and child table + tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG) + tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG) + + tdSql.error(f"create table {stable_name} (ts timestamp, pk int {pk_spell.value}, c2 int) tags (engine int)", show=SHOW_LOG) + + # create normal table + tdSql.error(f"create table {ntable_name} (ts timestamp, pk int {pk_spell.value}, c2 int)", show=SHOW_LOG) + + def test_update_pk(self, table_name: str, t_type: TableType): + # create super table and child table + tdSql.execute(f"drop table if exists {table_name}", show=SHOW_LOG) + + if t_type == TableType.SUPERTABLE: + tdSql.execute(f"create table {table_name} (ts timestamp, c2 int) tags (engine int)", show=SHOW_LOG) + elif t_type == TableType.NORNALTABLE: + # create normal table + tdSql.execute(f"create table {table_name} (ts timestamp, c2 int)", show=SHOW_LOG) + + tdSql.error(f"alter table {table_name} add column pk varchar(100) primary key", show=SHOW_LOG) + + tdSql.execute(f"drop table if exists {table_name}", show=SHOW_LOG) + if t_type == TableType.SUPERTABLE: + tdSql.execute(f"create table {table_name} (ts timestamp, pk varchar(200) primary key, c2 int) tags (engine int)", show=SHOW_LOG) + elif t_type == TableType.NORNALTABLE: + # create normal table + tdSql.execute(f"create table {table_name} (ts timestamp, pk varchar(200) primary key, c2 int)", show=SHOW_LOG) + + tdSql.error(f"alter table {table_name} add column new_pk varchar(20) primary key", show=SHOW_LOG) + for date_type in IllegalDataType.__members__.items(): + tdSql.error(f"alter table {table_name} modify column pk {date_type[1].value}", show=SHOW_LOG) + for date_type in LegalDataType.__members__.items(): + tdSql.error(f"alter table {table_name} modify column pk {date_type[1].value}", show=SHOW_LOG) + + tdSql.error(f"alter table {table_name} modify column pk varchar(300)", show=SHOW_LOG) + tdSql.error(f"alter table {table_name} rename column pk new_pk", show=SHOW_LOG) + tdSql.error(f"alter table {table_name} drop column pk", show=SHOW_LOG) + + def run(self): + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + # 1.check legal data type + for date_type in LegalDataType.__members__.items(): + self.test_pk_datatype_legal('s_table', 'c_table', 'n_table', date_type[1]) + + # 2.check illegal data type + for date_type in IllegalDataType.__members__.items(): + self.test_pk_datatype_illegal('s_table', 'n_table', date_type[1]) + + # 3.check legal spell + for date_type in LegalSpell.__members__.items(): + self.test_pk_spell_legal('s_table', 'n_table', date_type[1]) + + # 4.check illegal spell + for date_type in IllegalSpell.__members__.items(): + self.test_pk_spell_illegal('s_table', 'n_table', date_type[1]) + + # 5.only define ts and pk columns + # create super table and child table + tdSql.execute(f"drop table if exists s_table", show=SHOW_LOG) + tdSql.execute(f"drop table if exists n_table", show=SHOW_LOG) + + tdSql.execute(f"create table s_table (ts timestamp, pk int primary key) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table c_table using s_table tags (1)", show=SHOW_LOG) + tdSql.execute(f"insert into c_table values(now, 1)", show=SHOW_LOG) + tdSql.query(f"select * from s_table", show=SHOW_LOG) + tdSql.checkRows(1) + + # create normal table + tdSql.execute(f"create table n_table (ts timestamp, pk int primary key)", show=SHOW_LOG) + tdSql.execute(f"insert into n_table values(now, 1)", show=SHOW_LOG) + tdSql.query(f"select * from n_table", show=SHOW_LOG) + tdSql.checkRows(1) + + # 6.mutiple pk & pk not defined as sencod column + tdSql.execute(f"drop table if exists s_table", show=SHOW_LOG) + tdSql.execute(f"drop table if exists n_table", show=SHOW_LOG) + + # create super table + tdSql.error(f"create table s_table (ts timestamp, c1 int, pk1 int primary key) tags (engine int)", show=SHOW_LOG) + tdSql.error(f"create table s_table (ts timestamp, pk1 int primary key, pk2 int primary key) tags (engine int)", show=SHOW_LOG) + tdSql.error(f"create table s_table (ts timestamp, pk1 int primary key, c2 int, pk2 int primary key) tags (engine int)", show=SHOW_LOG) + # create normal table + tdSql.error(f"create table n_table (ts timestamp, c1 int, pk1 int primary key)", show=SHOW_LOG) + tdSql.error(f"create table n_table (ts timestamp, pk1 int primary key, pk2 int primary key)", show=SHOW_LOG) + tdSql.error(f"create table n_table (ts timestamp, pk1 int primary key, c2 int, pk2 int primary key)", show=SHOW_LOG) + + # 7.add\update\delete pk column is not support + self.test_update_pk('s_table', TableType.SUPERTABLE) + self.test_update_pk('n_table', TableType.NORNALTABLE) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/composite_primary_key_delete.py b/tests/system-test/1-insert/composite_primary_key_delete.py new file mode 100644 index 0000000000..31c7d7428e --- /dev/null +++ b/tests/system-test/1-insert/composite_primary_key_delete.py @@ -0,0 +1,244 @@ +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.csv import * +import os +import taos +import json +from taos import SmlProtocol, SmlPrecision +from taos.error import SchemalessError + + +class LegalDataType(Enum): + INT = 'INT' + UINT = 'INT UNSIGNED' + BIGINT = 'BIGINT' + UBIGINT = 'BIGINT UNSIGNED' + VARCHAR = 'VARCHAR(100)' + BINARY = 'BINARY(100)' + + +class TableType(Enum): + SUPERTABLE = 0 + CHILDTABLE = 1 + NORNALTABLE = 2 + +SHOW_LOG = True +STAET_TS = '2023-10-01 00:00:00.000' + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db_insert_composite_primary_key" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + self.testcasePath = os.path.split(__file__)[0] + self.testcasePath = self.testcasePath.replace('\\', '//') + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + + self.stable_name = 's_table' + self.ctable_name = 'c_table' + self.ntable_name = 'n_table' + self.ts_list = {} + + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database} CACHEMODEL 'none'") + tdSql.execute(f"use {self.database}") + + def get_latest_ts(self, table_name: str): + tdSql.query(f'select last(ts) from {table_name}') + now_time = tdSql.queryResult[0][0].strftime("%Y-%m-%d %H:%M:%S.%f") + return f"'{now_time}'" + + def prepare_data(self, dtype: LegalDataType): + # drop super table and child table + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}") + + # create super table & child table + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c1 smallint, c2 varchar(10)) tags (engine int)", show=SHOW_LOG) + + table_name1 = f'{self.ctable_name}_1' + tdSql.execute(f"insert into {table_name1} using {self.stable_name} tags(1) values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_1 = self.get_latest_ts(table_name1) + tdSql.execute(f"insert into {table_name1} values({child_ts_1}, 2, '8', '6') ({child_ts_1}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name1} (ts, pk, c2) values({child_ts_1}, 4, '8') ({child_ts_1}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name1} values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_2 = self.get_latest_ts(table_name1) + tdSql.execute(f"insert into {table_name1} values({child_ts_2}, 2, '8', '6') ({child_ts_2}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name1} (ts, pk, c2) values({child_ts_2}, 4, '8') ({child_ts_2}, 5, '9')", show=SHOW_LOG) + + table_name2 = f'{self.ctable_name}_2' + tdSql.execute(f"insert into {table_name2} using {self.stable_name} tags(2) values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_3 = self.get_latest_ts(table_name2) + tdSql.execute(f"insert into {table_name2} values({child_ts_3}, 2, '8', '6') ({child_ts_3}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name2} (ts, pk, c2) values({child_ts_3}, 4, '8') ({child_ts_3}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name2} values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_4 = self.get_latest_ts(table_name2) + tdSql.execute(f"insert into {table_name2} values({child_ts_4}, 2, '8', '6') ({child_ts_4}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name2} (ts, pk, c2) values({child_ts_4}, 4, '8') ({child_ts_4}, 5, '9')", show=SHOW_LOG) + + table_name3 = f'{self.ctable_name}_3' + tdSql.execute(f"insert into {table_name3} using {self.stable_name} tags(3) values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_5 = self.get_latest_ts(table_name3) + tdSql.execute(f"insert into {table_name3} values({child_ts_5}, 2, '8', '6') ({child_ts_5}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name3} (ts, pk, c2) values({child_ts_5}, 4, '8') ({child_ts_5}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name3} values(now, 1, '7', '6')", show=SHOW_LOG) + child_ts_6 = self.get_latest_ts(table_name3) + tdSql.execute(f"insert into {table_name3} values({child_ts_6}, 2, '8', '6') ({child_ts_6}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name3} (ts, pk, c2) values({child_ts_6}, 4, '8') ({child_ts_6}, 5, '9')", show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(30) + + # create normal table + tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c1 smallint, c2 varchar(10))", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_1 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_1}, 2, '8', '6') ({normal_ts_1}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_1}, 4, '8') ({normal_ts_1}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_2 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_2}, 2, '8', '6') ({normal_ts_2}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_2}, 4, '8') ({normal_ts_2}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_3 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_3}, 2, '8', '6') ({normal_ts_3}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_3}, 4, '8') ({normal_ts_3}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_4 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_4}, 2, '8', '6') ({normal_ts_4}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_4}, 4, '8') ({normal_ts_4}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_5 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_5}, 2, '8', '6') ({normal_ts_5}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_5}, 4, '8') ({normal_ts_5}, 5, '9')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG) + normal_ts_6 = self.get_latest_ts(self.ntable_name) + tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_6}, 2, '8', '6') ({normal_ts_6}, 3, '9', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_6}, 4, '8') ({normal_ts_6}, 5, '9')", show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(30) + + self.ts_list['child_ts_1'] = child_ts_1 + self.ts_list['child_ts_2'] = child_ts_2 + self.ts_list['child_ts_3'] = child_ts_3 + self.ts_list['child_ts_4'] = child_ts_4 + self.ts_list['child_ts_5'] = child_ts_5 + self.ts_list['child_ts_6'] = child_ts_6 + self.ts_list['normal_ts_1'] = normal_ts_1 + self.ts_list['normal_ts_2'] = normal_ts_2 + self.ts_list['normal_ts_3'] = normal_ts_3 + self.ts_list['normal_ts_4'] = normal_ts_4 + self.ts_list['normal_ts_5'] = normal_ts_5 + self.ts_list['normal_ts_6'] = normal_ts_6 + + + + def test_delete_data(self): + # delete with ts + tdSql.execute(f"delete from {self.stable_name} where ts={self.ts_list['child_ts_1']} ", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(25) + tdSql.execute(f"delete from {self.ctable_name}_1 where ts={self.ts_list['child_ts_2']} ", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(20) + tdSql.execute(f"delete from {self.ntable_name} where ts={self.ts_list['normal_ts_1']} ", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(25) + + # delete with ts range + tdSql.execute(f"delete from {self.stable_name} where ts>{self.ts_list['child_ts_2']} and ts<{self.ts_list['child_ts_4']}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(15) + tdSql.execute(f"delete from {self.ctable_name}_2 where ts>{self.ts_list['child_ts_2']} and ts<{self.ts_list['child_ts_5']}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(10) + tdSql.execute(f"delete from {self.ntable_name} where ts>{self.ts_list['normal_ts_2']} and ts<{self.ts_list['normal_ts_5']}", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(15) + + tdSql.execute(f"delete from {self.stable_name}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + + table_name4 = f'{self.ctable_name}_4' + tdSql.execute(f"insert into {table_name4} using {self.stable_name} tags(3) values(now, 1, '7', '6')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name4} values(now+1s, 2, '8', '6') (now+2s, 3, '9', '6')", show=SHOW_LOG) + + tdSql.execute(f"delete from {table_name4}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + tdSql.execute(f"delete from {self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + + tdSql.execute(f"delete from {self.stable_name}", show=SHOW_LOG) + tdSql.execute(f"delete from {table_name4}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + tdSql.execute(f"delete from {self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + + def test_delete_data_illegal(self, dtype: LegalDataType): + if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + pk_condition_value = '\'1\'' + else: + pk_condition_value = '1' + # delete with ts & pk + tdSql.error(f"delete from {self.stable_name} where ts={self.ts_list['child_ts_1']} and pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ctable_name}_2 where ts={self.ts_list['child_ts_1']} and pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ntable_name} where ts={self.ts_list['normal_ts_1']} and pk={pk_condition_value}", show=SHOW_LOG) + + # delete with ts range & pk + tdSql.error(f"delete from {self.stable_name} where ts>{self.ts_list['child_ts_1']} and ts<{self.ts_list['child_ts_2']} and pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ctable_name}_2 where ts>{self.ts_list['child_ts_1']} and ts<{self.ts_list['child_ts_2']} and pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ntable_name} where ts>{self.ts_list['normal_ts_1']} and ts<{self.ts_list['normal_ts_2']} and pk={pk_condition_value}", show=SHOW_LOG) + + # delete with pk + tdSql.error(f"delete from {self.stable_name} where pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ctable_name}_2 where pk={pk_condition_value}", show=SHOW_LOG) + tdSql.error(f"delete from {self.ntable_name} where pk={pk_condition_value}", show=SHOW_LOG) + + def _compare_table_data(self, result1, result2, row = 0, col = 0): + for i in range(row): + for j in range(col): + if result1[i][j] != result2[i][j]: + tdSql.checkEqual(False, True) + + def run(self): + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + for date_type in LegalDataType.__members__.items(): + self.prepare_data(date_type[1]) + self.test_delete_data_illegal(date_type[1]) + self.test_delete_data() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/composite_primary_key_insert.py b/tests/system-test/1-insert/composite_primary_key_insert.py new file mode 100644 index 0000000000..1a22eb781c --- /dev/null +++ b/tests/system-test/1-insert/composite_primary_key_insert.py @@ -0,0 +1,958 @@ +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.csv import * +import os +import taos +import json +from taos import SmlProtocol, SmlPrecision +from taos.error import SchemalessError + + +class IllegalData(Enum): + NULL = 'null' + # EMPTY = '\'\'' + NONE = 'none' + # TRUE = 'true' + # FALSE = 'false' + +# class IllegalDataVar(Enum): +# NULL = 'null' +# EMPTY = '\'\'' +# NONE = 'none' +# TRUE = 'true' +# FALSE = 'false' + + +class LegalDataType(Enum): + INT = 'INT' + UINT = 'INT UNSIGNED' + BIGINT = 'BIGINT' + UBIGINT = 'BIGINT UNSIGNED' + VARCHAR = 'VARCHAR(100)' + BINARY = 'BINARY(100)' + + +class TableType(Enum): + SUPERTABLE = 0 + CHILDTABLE = 1 + NORNALTABLE = 2 + +class HasPK(Enum): + NO = 0 + YES = 1 + +SHOW_LOG = True +STAET_TS = '2023-10-01 00:00:00.000' + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + self.database = "db_insert_composite_primary_key" + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + self.testcasePath = os.path.split(__file__)[0] + self.testcasePath = self.testcasePath.replace('\\', '//') + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + # tdSql.execute(f"insert into db4096.ctb00 file '{self.testcasePath}//tableColumn4096csvLength64k.csv'") + + self.tdCsv = TDCsv() + self.tdCsv.file_path = self.testcasePath + self.tdCsv.file_name = 'file1.csv' + + self.stable_name = 's_table' + self.ctable_name = 'c_table' + self.ntable_name = 'n_table' + + + def prepare_db(self): + tdSql.execute(f"drop database if exists {self.database}") + tdSql.execute(f"create database {self.database}") + tdSql.execute(f"use {self.database}") + + def get_latest_ts(self, table_name: str): + tdSql.query(f'select last(ts) from {table_name}') + now_time = tdSql.queryResult[0][0].strftime("%Y-%m-%d %H:%M:%S.%f") + return f"'{now_time}'" + + def test_insert_data(self, dtype: LegalDataType, hasPk: HasPK): + # drop super table and child table + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}_1") + tdSql.execute(f"drop table if exists {self.ntable_name}_2") + if hasPk: + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value}) tags (engine int)", show=SHOW_LOG) + else: + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value}, c3 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + # 1.insert into value through supper table + table_name = f'{self.ctable_name}_1' + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now, 1, '1')", show=SHOW_LOG) + current_ts1 = self.get_latest_ts(self.stable_name) + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts1}, 2, '2')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts1}, 3, '3')", show=SHOW_LOG) + + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now + 1s, 1, '4')", show=SHOW_LOG) + current_ts2 = self.get_latest_ts(self.stable_name) + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts2}, 2, '5')", show=SHOW_LOG) + tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk) values('{table_name}', 1, now + 2s, 2)", show=SHOW_LOG) + + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts1}", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts2}", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts2} and pk = 2", show=SHOW_LOG) + tdSql.checkRows(1) + self._check_select(self.stable_name) + + # 2.insert into value through child table + table_name = f'{self.ctable_name}_2' + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) values(now, 1, '7', '6')", show=SHOW_LOG) + current_ts3 = self.get_latest_ts(table_name) + tdSql.execute(f"insert into {table_name} values({current_ts3}, 2, '8', '6') ({current_ts3}, 3, '9', '6')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) values(now + 2s, 1, '10', '6')", show=SHOW_LOG) + current_ts4 = self.get_latest_ts(table_name) + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) (ts, pk, c2) values({current_ts4}, 2, '11')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) (ts, pk) values(now + 4s, 2)", show=SHOW_LOG) + + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts3}", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts4}", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts4} and pk = 2", show=SHOW_LOG) + tdSql.checkRows(1) + self._check_select(table_name) + + # 3.insert value into child table from csv file + data = [ + ['ts','pk','c2'], + ['\'2024-03-29 16:55:42.572\'','1','1','100'], + ['\'2024-03-29 16:55:42.572\'','2','2','100'], + ['\'2024-03-29 16:55:42.572\'','3','3','100'], + ['\'2024-03-29 16:55:43.586\'','1','4','100'], + ['\'2024-03-29 16:55:43.586\'','2','5','100'], + ['\'2024-03-29 16:55:44.595\'','2','6','100'] + ] + + self.tdCsv.delete() + self.tdCsv.write(data) + + table_name = f'{self.ctable_name}_3' + tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:42.572'", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:43.586'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:43.586' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + self._check_select(table_name) + + # 4.insert value into child table from csv file, create table automatically + data = [ + ['ts','pk','c2'], + ['\'2024-03-28 16:55:42.572\'','1','1','100'], + ['\'2024-03-28 16:55:42.572\'','2','2','100'], + ['\'2024-03-28 16:55:42.572\'','3','3','100'], + ['\'2024-03-28 16:55:43.586\'','1','4','100'], + ['\'2024-03-28 16:55:43.586\'','2','5','100'], + ['\'2024-03-28 16:55:44.595\'','2','6','100'] + ] + + self.tdCsv.delete() + self.tdCsv.write(data) + + table_name = f'{self.ctable_name}_4' + self._check_select(self.stable_name) + tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(4) file '{self.tdCsv.file}'", show=SHOW_LOG) + + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:42.572'", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:43.586'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:43.586' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + self._check_select(self.stable_name) + + # 5.insert value into normal table from csv file + table_name = f'{self.ntable_name}_1' + if hasPk: + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + else: + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:42.572'", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:43.586'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:43.586' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + self._check_select(table_name) + + # 6.insert value into normal table + table_name = f'{self.ntable_name}_2' + if hasPk: + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + else: + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name} values(now, 1, '1', '234')", show=SHOW_LOG) + current_ts1 = self.get_latest_ts(table_name) + tdSql.execute(f"insert into {table_name} values({current_ts1}, 2, '2', '234') ({current_ts1}, 3, '3', '234')", show=SHOW_LOG) + + tdSql.execute(f"insert into {table_name} values(now + 1s, 1, '4', '234')", show=SHOW_LOG) + current_ts2 = self.get_latest_ts(table_name) + tdSql.execute(f"insert into {table_name} (ts, pk, c2) values({current_ts2}, 2, '5')", show=SHOW_LOG) + tdSql.execute(f"insert into {table_name} (ts, pk) values(now + 2s, 2)", show=SHOW_LOG) + + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(6) + tdSql.query(f"select * from {table_name} where ts ={current_ts1}", show=SHOW_LOG) + tdSql.checkRows(3) + tdSql.query(f"select * from {table_name} where ts ={current_ts2}", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {table_name} where ts ={current_ts2} and pk = 2", show=SHOW_LOG) + tdSql.checkRows(1) + self._check_select(table_name) + + def test_insert_data_illegal(self, dtype: LegalDataType, illegal_data: IllegalData): + # drop tables + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}_1") + tdSql.execute(f"drop table if exists {self.ntable_name}_2") + + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + # 1.insert into value through supper table + table_name = f'{self.ctable_name}_1' + tdSql.error(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now, {illegal_data.value}, '1')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + # 2.insert into value through child table + table_name = f'{self.ctable_name}_2' + tdSql.error(f"insert into {table_name} using {self.stable_name} tags(2) values(now, {illegal_data.value}, '7')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + # 4.insert value into child table from csv file + data = [ + ['ts','pk','c2'], + ['\'2024-03-29 16:55:42.572\'','1','1'], + ['\'2024-03-29 16:55:42.572\'',f'{illegal_data.value}','2'], + ['\'2024-03-29 16:55:42.572\'','3','3'], + ['\'2024-03-29 16:55:43.586\'','1','4'], + ['\'2024-03-29 16:55:43.586\'','2','5'], + ['\'2024-03-29 16:55:44.595\'','2','6'] + ] + + self.tdCsv.delete() + self.tdCsv.write(data) + + table_name = f'{self.ctable_name}_3' + tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(0) + self._check_select(table_name) + + # 5.insert value into child table from csv file, create table automatically + table_name = f'{self.ctable_name}_4' + tdSql.error(f"insert into {table_name} using {self.stable_name} tags(4) file '{self.tdCsv.file}'", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + # 6.insert value into normal table from csv file + table_name = f'{self.ntable_name}_1' + tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(0) + self._check_select(table_name) + + # 7.insert value into normal table + table_name = f'{self.ntable_name}_2' + tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + tdSql.error(f"insert into {table_name} values(now, {illegal_data.value}, '1')", show=SHOW_LOG) + tdSql.query(f'select * from {table_name}') + tdSql.checkRows(0) + self._check_select(table_name) + + def test_insert_select(self, dtype: LegalDataType): + # # 1.pk table to non-pk table, throw error + tdSql.execute(f"drop table if exists source_{self.stable_name}") + tdSql.execute(f"drop table if exists source_{self.ctable_name}") + tdSql.execute(f"drop table if exists source_{self.ntable_name}") + tdSql.execute(f"drop table if exists dest_{self.stable_name}") + tdSql.execute(f"drop table if exists dest_{self.ntable_name}") + + tdSql.execute(f"create table source_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table source_{self.ctable_name} using source_{self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.execute(f"create table source_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"create table dest_{self.ntable_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG) + + tdSql.error(f"insert into dest_{self.ntable_name} select * from source_{self.stable_name})", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ntable_name} select * from source_{self.ctable_name})", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ntable_name} select * from source_{self.ntable_name})", show=SHOW_LOG) + + # 2.non-pk table to pk table + tdSql.execute(f"drop table if exists source_{self.stable_name}") + tdSql.execute(f"drop table if exists source_{self.ctable_name}") + tdSql.execute(f"drop table if exists source_{self.ntable_name}") + tdSql.execute(f"drop table if exists dest_{self.stable_name}") + tdSql.execute(f"drop table if exists dest_{self.ntable_name}") + + # create dest super & child table + tdSql.execute(f"create table dest_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table dest_{self.ctable_name} using dest_{self.stable_name} tags(3)", show=SHOW_LOG) + # tdSql.execute(f"insert into source_{self.ctable_name} values(now, 1, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) + + # create normal dest table + tdSql.execute(f"create table dest_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + + # create source table & insert data + tdSql.execute(f"create table source_{self.ntable_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"insert into source_{self.ntable_name} values(now, 1, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) + tdSql.query(f'select * from source_{self.ntable_name}') + source_data = tdSql.queryResult + + tdSql.execute(f"insert into dest_{self.ctable_name} select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from dest_{self.ctable_name}') + dest_data = tdSql.queryResult + self._compare_table_data(source_data, dest_data, 3, 3) + self._check_select(f'dest_{self.ctable_name}') + tdSql.execute(f"delete from dest_{self.ctable_name}", show=SHOW_LOG) + self._check_select(f'dest_{self.ctable_name}') + + tdSql.execute(f"insert into dest_{self.ntable_name} select * from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from dest_{self.ntable_name}') + dest_data = tdSql.queryResult + self._compare_table_data(source_data, dest_data, 3, 3) + self._check_select(f'dest_{self.ntable_name}') + tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG) + self._check_select(f'dest_{self.ntable_name}') + + # TD-29363 + tdSql.execute(f"drop table if exists source_null") + + tdSql.execute(f"create table source_null (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"insert into source_null values(now, null, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ntable_name} values(now, null, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ctable_name} select ts, pk, c2 from source_null", show=SHOW_LOG) + tdSql.error(f"insert into dest_{self.ntable_name} select * from source_null", show=SHOW_LOG) + + # 3.pk table to pk table + tdSql.execute(f"drop table if exists source_{self.stable_name}") + tdSql.execute(f"drop table if exists source_{self.ctable_name}") + tdSql.execute(f"drop table if exists source_{self.ntable_name}") + tdSql.execute(f"drop table if exists dest_{self.stable_name}") + tdSql.execute(f"drop table if exists dest_{self.ntable_name}") + + # create dest super & child table + tdSql.execute(f"create table dest_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value}) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table dest_{self.ctable_name} using dest_{self.stable_name} tags(3)", show=SHOW_LOG) + tdSql.execute(f"insert into dest_{self.ctable_name} values('2024-04-01 17:38:08.764', 1, 1, 100) ('2024-04-01 17:38:08.764', 2, 2, 200) ('2024-04-01 17:38:08.764', 3, 3, 300)", show=SHOW_LOG) + tdSql.execute(f"insert into dest_{self.ctable_name} values('2024-04-02 17:38:08.764', 1, 4, 400) ('2024-04-02 17:38:08.764', 2, 1, 500) ('2024-04-02 17:38:08.764', 3, 1, 600)", show=SHOW_LOG) + + # create normal dest table + tdSql.execute(f"create table dest_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"insert into dest_{self.ntable_name} values('2024-04-01 17:38:08.764', 1, 1, 100) ('2024-04-01 17:38:08.764', 2, 2, 200) ('2024-04-01 17:38:08.764', 3, 3, 300)", show=SHOW_LOG) + tdSql.execute(f"insert into dest_{self.ntable_name} values('2024-04-02 17:38:08.764', 1, 4, 400) ('2024-04-02 17:38:08.764', 2, 1, 500) ('2024-04-02 17:38:08.764', 3, 1, 600)", show=SHOW_LOG) + + # create source table & insert data + tdSql.execute(f"create table source_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"insert into source_{self.ntable_name} values('2024-04-02 17:38:08.764', 1, 4, 800) ('2024-04-02 17:38:08.764', 2, 5, 400) ('2024-04-02 17:38:08.764', 3, 6, 600)", show=SHOW_LOG) + tdSql.execute(f"insert into source_{self.ntable_name} values('2024-04-04 17:38:08.764', 1, 7, 365)", show=SHOW_LOG) + + # insert into select + tdSql.execute(f"insert into dest_{self.ctable_name} (ts, pk, c2) select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from dest_{self.ctable_name}') + tdSql.checkRows(7) + if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + tdSql.query(f"select * from dest_{self.ctable_name} where c2='5' or c2='6'", show=SHOW_LOG) + else: + tdSql.query(f'select * from dest_{self.ctable_name} where c2=5 or c2=6', show=SHOW_LOG) + tdSql.checkRows(2) + self._check_select(f'dest_{self.ctable_name}') + tdSql.execute(f"delete from dest_{self.ctable_name}", show=SHOW_LOG) + self._check_select(f'dest_{self.ctable_name}') + + tdSql.execute(f"insert into dest_{self.ntable_name} (ts, pk, c2) select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG) + tdSql.query(f'select * from dest_{self.ntable_name}') + tdSql.checkRows(7) + if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + tdSql.query(f"select * from dest_{self.ntable_name} where c2='5' or c2='6'") + else: + tdSql.query(f'select * from dest_{self.ntable_name} where c2=5 or c2=6') + tdSql.checkRows(2) + self._check_select(f'dest_{self.ntable_name}') + tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG) + self._check_select(f'dest_{self.ntable_name}') + + def test_schemaless_error(self): + # 5.1.insert into values via influxDB + lines = ["meters,location=California.LosAngeles,groupid=2 current=11i32,voltage=221,phase=0.28 1648432611249000", + "meters,location=California.LosAngeles,groupid=2 current=13i32,voltage=223,phase=0.29 1648432611249000", + "meters,location=California.LosAngeles,groupid=3 current=10i32,voltage=223,phase=0.29 1648432611249300", + "meters,location=California.LosAngeles,groupid=3 current=11i32,voltage=221,phase=0.35 1648432611249300", + ] + try: + conn = taos.connect() + conn.execute("drop database if exists influxDB") + conn.execute("CREATE DATABASE influxDB precision 'us'") + conn.execute("USE influxDB") + conn.execute("CREATE STABLE `meters` (`_ts` TIMESTAMP, `current` int primary key, `voltage` DOUBLE, `phase` DOUBLE) TAGS (`location` NCHAR(32), `groupid` NCHAR(2))") + conn.execute("create table t_be97833a0e1f523fcdaeb6291d6fdf27 using meters tags('California.LosAngeles', 2)") + conn.execute("create table t_10b65f71ff8970369c8c18de0d6be028 using meters tags('California.LosAngeles', 3)") + conn.schemaless_insert(lines, SmlProtocol.LINE_PROTOCOL, SmlPrecision.MICRO_SECONDS) + tdSql.checkEqual(False, True) + except SchemalessError as errMsg: + tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True) + + # 5.2.insert into values via OpenTSDB + lines = ["meters.current 1648432611249 10i32 location=California.SanFrancisco groupid=2", + "meters.current 1648432611250 12i32 location=California.SanFrancisco groupid=2", + "meters.current 1648432611249 10i32 location=California.LosAngeles groupid=3", + "meters.current 1648432611250 11i32 location=California.LosAngeles groupid=3", + "meters.voltage 1648432611249 219i32 location=California.SanFrancisco groupid=2", + "meters.voltage 1648432611250 218i32 location=California.SanFrancisco groupid=2", + "meters.voltage 1648432611249 221i32 location=California.LosAngeles groupid=3", + "meters.voltage 1648432611250 217i32 location=California.LosAngeles groupid=3", + ] + try: + conn = taos.connect() + conn.execute("drop database if exists OpenTSDB") + conn.execute("CREATE DATABASE OpenTSDB precision 'us'") + conn.execute("USE OpenTSDB") + conn.execute("CREATE STABLE `meters_current` (`_ts` TIMESTAMP, `_value` INT primary key) TAGS (`location` NCHAR(32), `groupid` NCHAR(2))") + conn.execute("CREATE TABLE `t_c66ea0b2497be26ca9d328b59c39dd61` USING `meters_current` (`location`, `groupid`) TAGS ('California.LosAngeles', '3')") + conn.execute("CREATE TABLE `t_e71c6cf63cfcabb0e261886adea02274` USING `meters_current` (`location`, `groupid`) TAGS ('California.SanFrancisco', '2')") + conn.schemaless_insert(lines, SmlProtocol.TELNET_PROTOCOL, SmlPrecision.NOT_CONFIGURED) + tdSql.checkEqual(False, True) + except SchemalessError as errMsg: + tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True) + + # 5.3.insert into values via OpenTSDB Json + lines = [{"metric": "meters.current", "timestamp": 1648432611249, "value": "a32", "tags": {"location": "California.SanFrancisco", "groupid": 2}}] + try: + conn = taos.connect() + conn.execute("drop database if exists OpenTSDBJson") + conn.execute("CREATE DATABASE OpenTSDBJson") + conn.execute("USE OpenTSDBJson") + conn.execute("CREATE STABLE `meters_current` (`_ts` TIMESTAMP, `_value` varchar(10) primary key) TAGS (`location` VARCHAR(32), `groupid` DOUBLE)") + conn.execute("CREATE TABLE `t_71d176bfc4c952b64d30d719004807a0` USING `meters_current` (`location`, `groupid`) TAGS ('California.SanFrancisco', 2.000000e+00)") + # global lines + lines = json.dumps(lines) + # note: the first parameter must be a list with only one element. + conn.schemaless_insert([lines], SmlProtocol.JSON_PROTOCOL, SmlPrecision.NOT_CONFIGURED) + tdSql.checkEqual(False, True) + except SchemalessError as errMsg: + tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True) + + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + + def test_insert_values_special(self, dtype: LegalDataType): + # 4.insert into values without ts column + # drop tables + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}") + + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 binary(10)) tags (engine int)", show=SHOW_LOG) + + # # 4.1.insert into value through supper table + tdSql.error(f"insert into {self.stable_name} (tbname, engine, pk, c2) values('{self.ctable_name}', 1, '1', '1')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + # # 4.2.insert into value through child table + tdSql.error(f"insert into {self.ctable_name} using {self.stable_name} tags(2) (pk, c2) values('1', '7')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + # # 4.3.insert value into normal table + tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG) + tdSql.error(f"insert into {self.ntable_name} (pk, c2) values('1', '1')", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + self._check_select(self.ntable_name) + + # 5.insert into values without pk column + # drop tables + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}") + + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 binary(10)) tags (engine int)", show=SHOW_LOG) + + # # 5.1.insert into value through supper table + tdSql.error(f"insert into {self.stable_name} (tbname, engine, ts, c2) values('{self.ctable_name}', 1, now, '1')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + # # 5.2.insert into value through child table + tdSql.error(f"insert into {self.ctable_name} using {self.stable_name} tags(2) (ts, c2) values(now, '7')", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + # # 5.3.insert value into normal table + tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG) + tdSql.error(f"insert into {self.ntable_name} (ts, c2) values(now, '1')", show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + self._check_select(self.ntable_name) + + def test_insert_into_mutiple_tables(self, dtype: LegalDataType): + # drop super table and child table + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + # 1.insert into value by supper table syntax + error_sql = f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) " \ + f"values('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 1, 100) " \ + f"('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 2, 200) " \ + f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 1, 300) " \ + f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 2, 400) " \ + f"('{self.ctable_name}_3', 3, '2021-07-15 14:06:34.630', null, 500)" + + tdSql.error(error_sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + sql = f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) " \ + f"values('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 1, 100) " \ + f"('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 2, 200) " \ + f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 1, 300) " \ + f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 2, 400) " \ + f"('{self.ctable_name}_3', 3, '2021-07-15 14:06:34.630', 1, 500)" + + tdSql.execute(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(5) + tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + self._check_select(self.stable_name) + + # 2.insert into value and create table automatically + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + error_sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " \ + f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) values('2021-07-15 14:06:34.630', null, 500)" + + tdSql.error(error_sql, show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " \ + f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) values('2021-07-15 14:06:34.630', 1, 500)" + + tdSql.execute(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(5) + tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630'", show=SHOW_LOG) + tdSql.checkRows(2) + tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + self._check_select(self.stable_name) + + # 3.insert value into child table from csv file, create table automatically + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + + error_data = [ + ['ts','pk','c2'], + ['\'2024-03-29 16:55:42.572\'','1','1'], + ['\'2024-03-29 16:55:42.572\'','2','2'], + ['\'2024-03-29 16:55:42.572\'','null','3'], + ['\'2024-03-29 16:55:43.586\'','1','4'], + ['\'2024-03-29 16:55:43.586\'','2','5'], + ['\'2024-03-29 16:55:44.595\'','2','6'] + ] + + self.tdCsv.delete() + self.tdCsv.write(error_data) + + sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) FILE '{self.tdCsv.file}' " \ + f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) FILE '{self.tdCsv.file}' " \ + f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) FILE '{self.tdCsv.file}'" + + tdSql.error(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + self._check_select(self.stable_name) + + data = [ + ['ts','pk','c2'], + ['\'2024-03-29 16:55:42.572\'','1','1'], + ['\'2024-03-29 16:55:42.572\'','2','2'], + ['\'2024-03-29 16:55:42.572\'','2','3'], + ['\'2024-03-29 16:55:43.586\'','1','4'], + ['\'2024-03-29 16:55:43.586\'','2','5'], + ['\'2024-03-29 16:55:44.595\'','2','6'] + ] + + self.tdCsv.delete() + self.tdCsv.write(data) + + sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) FILE '{self.tdCsv.file}'" \ + f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) FILE '{self.tdCsv.file}'" \ + f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) FILE '{self.tdCsv.file}'" + + tdSql.execute(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(15) + tdSql.query(f"select * from {self.stable_name} where ts='2024-03-29 16:55:42.572'", show=SHOW_LOG) + tdSql.checkRows(6) + tdSql.query(f"select * from {self.stable_name} where ts='2024-03-29 16:55:42.572' and pk=2", show=SHOW_LOG) + tdSql.checkRows(3) + self._check_select(self.stable_name) + + # 6.insert value into normal table + tdSql.execute(f"drop table if exists {self.ntable_name}_1") + tdSql.execute(f"drop table if exists {self.ntable_name}_2") + tdSql.execute(f"create table {self.ntable_name}_1 (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + tdSql.execute(f"create table {self.ntable_name}_2 (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + + sql = f"insert into {self.ntable_name}_1 values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ntable_name}_2 values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " + + tdSql.execute(sql, show=SHOW_LOG) + + tdSql.query(f'select * from {self.ntable_name}_1') + tdSql.checkRows(2) + tdSql.query(f"select * from {self.ntable_name}_2 where ts='2021-07-14 14:06:34.630' and pk=2", show=SHOW_LOG) + tdSql.checkRows(1) + self._check_select(self.ntable_name) + + # 7. insert value into child and normal table + tdSql.execute(f"drop table if exists {self.stable_name}") + tdSql.execute(f"drop table if exists {self.ctable_name}") + tdSql.execute(f"drop table if exists {self.ntable_name}") + + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG) + tdSql.execute(f"create table {self.ctable_name} using {self.stable_name} tags (1)", show=SHOW_LOG) + tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + + error_sql = f"insert into {self.ctable_name} values('2021-07-13 14:06:34.630', null, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ntable_name} values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " + + sql = f"insert into {self.ctable_name} values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \ + f"{self.ntable_name} values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " + + tdSql.error(error_sql, show=SHOW_LOG) + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(0) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(0) + + tdSql.execute(sql, show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(2) + tdSql.query(f'select * from {self.ntable_name}') + tdSql.checkRows(2) + self._check_select(self.stable_name) + self._check_select(self.ntable_name) + + def test_stmt(self, dtype: LegalDataType): + tdSql.execute(f"drop table if exists {self.stable_name}", show=SHOW_LOG) + tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 float) tags (engine int)", show=SHOW_LOG) + + sql = f"INSERT INTO ? USING {self.stable_name} TAGS(?) VALUES (?,?,?,?)" + + conn = taos.connect() + conn.select_db(self.database) + stmt = conn.statement(sql) + + tbname = f"d1001" + + tags = taos.new_bind_params(1) + tags[0].int([2]) + + stmt.set_tbname_tags(tbname, tags) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589, 1626861392589, 1626861392592)) + if dtype == LegalDataType.INT : + params[1].int((10, 12, 12)) + params[2].int([194, 200, 201]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((10, 12, 12)) + params[2].int_unsigned([194, 200, 201]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((10, 12, 12)) + params[2].bigint([194, 200, 201]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((10, 12, 12)) + params[2].bigint_unsigned([194, 200, 201]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary(("s10", "s12", "s12")) + params[2].binary(["s194", "s200", "s201"]) + params[3].float([0.31, 0.33, 0.31]) + + stmt.bind_param_batch(params) + + stmt.execute() + + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(3) + self._check_select(self.stable_name) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589)) + if dtype == LegalDataType.INT : + params[1].int((11)) + params[2].int([199]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((11)) + params[2].int_unsigned([199]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((11)) + params[2].bigint([199]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((11)) + params[2].bigint_unsigned([199]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary(("s11")) + params[2].binary(["s199"]) + params[3].float([0.31]) + + stmt.bind_param(params) + + stmt.execute() + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(4) + self._check_select(self.stable_name) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589)) + if dtype == LegalDataType.INT : + params[1].int((11)) + params[2].int([1000]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((11)) + params[2].int_unsigned([1000]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((11)) + params[2].bigint([1000]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((11)) + params[2].bigint_unsigned([1000]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary(("s11")) + params[2].binary(["1000"]) + params[3].float([0.31]) + + stmt.bind_param(params) + + stmt.execute() + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(4) + self._check_select(self.stable_name) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589)) + if dtype == LegalDataType.INT : + params[1].int(None) + params[2].int([199]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned(None) + params[2].int_unsigned([199]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint(None) + params[2].bigint([199]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned(None) + params[2].bigint_unsigned([199]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary(None) + params[2].binary(["s199"]) + params[3].float([0.31]) + + try: + stmt.bind_param(params) + tdSql.checkEqual(False, True) + except Exception as errMsg: + pass + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(4) + self._check_select(self.stable_name) + + params = taos.new_bind_params(4) + params[0].timestamp((1626861392589, 1626861392589, )) + if dtype == LegalDataType.INT : + params[1].int((None, 18,)) + params[2].int([194, 200]) + elif dtype == LegalDataType.UINT: + params[1].int_unsigned((None, 18)) + params[2].int_unsigned([194, 200]) + elif dtype == LegalDataType.BIGINT: + params[1].bigint((None, 18)) + params[2].bigint([194, 200]) + elif dtype == LegalDataType.UBIGINT: + params[1].bigint_unsigned((None, 18)) + params[2].bigint_unsigned([194, 200]) + elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + params[1].binary((None, "s18")) + params[2].binary(["s194", "s200"]) + params[3].float([0.31, 0.33, 0.31]) + + try: + stmt.bind_param(params) + tdSql.checkEqual(False, True) + except Exception as errMsg: + pass + + tdSql.query(f'select * from {self.stable_name}') + tdSql.checkRows(4) + self._check_select(self.stable_name) + + if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY: + tdSql.query(f'select * from {self.stable_name} where pk="s11"') + tdSql.checkEqual(tdSql.queryResult[0][2] == '1000', True) + else: + tdSql.query(f'select * from {self.stable_name} where pk=11') + tdSql.checkEqual(tdSql.queryResult[0][2] == 1000, True) + stmt.close() + + def test_implicit_conversion(self, dtype: LegalDataType): + + tdSql.execute(f"drop table if exists dest_table", show=SHOW_LOG) + tdSql.execute(f"create table dest_table (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG) + for type in LegalDataType: + if type == dtype: + continue + tdSql.execute(f"drop table if exists source_table", show=SHOW_LOG) + tdSql.execute(f"create table source_table (ts timestamp, pk {dtype.value} primary key, c2 int)", show=SHOW_LOG) + tdSql.execute(f"insert into source_table values(now, 100, 1000)", show=SHOW_LOG) + tdSql.execute(f"insert into dest_table select * from source_table", show=SHOW_LOG) + + tdSql.execute(f"flush database {self.database}", show=SHOW_LOG) + tdSql.query(f'select * from dest_table where c2=1000') + tdSql.checkRows(1) + tdSql.execute(f"delete from dest_table", show=SHOW_LOG) + self._check_select('dest_table') + + def _compare_table_data(self, result1, result2, row = 0, col = 0): + for i in range(row): + for j in range(col): + if result1[i][j] != result2[i][j]: + tdSql.checkEqual(False, True) + + def _check_select(self, table_nam: str): + tdSql.query(f'select count(*) from {table_nam} ') + tdSql.query(f'select * from {table_nam}') + tdSql.query(f'select last_row(*) from {table_nam}') + tdSql.query(f'select last_row(*) from {table_nam} group by tbname') + tdSql.query(f'select first(*) from {table_nam}') + tdSql.query(f'select first(*) from {table_nam} group by tbname') + tdSql.query(f'select last(*) from {table_nam}') + tdSql.query(f'select last(*) from {table_nam} group by tbname') + tdSql.query(f'select ts, last(pk) from {table_nam} order by ts') + + tdSql.query(f'select * from {table_nam} order by ts asc') + tdSql.query(f'select * from {table_nam} order by ts desc') + tdSql.query(f'select * from {table_nam} order by pk asc') + tdSql.query(f'select * from {table_nam} order by pk desc') + tdSql.query(f'select * from {table_nam} order by ts asc, pk desc') + tdSql.query(f'select * from {table_nam} order by ts desc, pk asc') + + def run(self): + tdSql.prepare(replica = self.replicaVar) + self.prepare_db() + + for date_type in LegalDataType.__members__.items(): + tdLog.info(f'') + # 1.insert into value with pk - pass + tdLog.info('[1.insert into value with pk]') + self.test_insert_data(date_type[1], HasPK.YES) + + # 2.insert into value without pk - pass + tdLog.info('[2.insert into value without pk]') + self.test_insert_data(date_type[1], HasPK.NO) + + # 3.insert into illegal data - pass + tdLog.info('[3.insert into illegal data]') + for illegal_data in IllegalData.__members__.items(): + self.test_insert_data_illegal(date_type[1], illegal_data[1]) + + # 4. insert into select - pass + tdLog.info('[4. insert into select]') + self.test_insert_select(date_type[1]) + + # 5. insert into values special cases - pass + tdLog.info('[5. insert into values special cases]') + self.test_insert_values_special(date_type[1]) + + # 6. test implicit conversion - pass + tdLog.info('[6. test implicit conversion]') + self.test_implicit_conversion(date_type[1]) + + # 7. insert into value to mutiple tables - pass + tdLog.info('[7. insert into value to mutiple tables]') + self.test_insert_into_mutiple_tables(date_type[1]) + + # 8. stmt wait for test!!!! + tdLog.info('[8. stmt wait for test]') + self.test_stmt(date_type[1]) + + # 9. insert data by schemaless model is not allowed - pass + tdLog.info('[9. insert data by schemaless model is not allowed]') + self.test_schemaless_error() + # while(True): + # self.test_stmt(LegalDataType.VARCHAR) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmq_primary_key.py b/tests/system-test/7-tmq/tmq_primary_key.py index 44de5466a8..8f62dc8783 100644 --- a/tests/system-test/7-tmq/tmq_primary_key.py +++ b/tests/system-test/7-tmq/tmq_primary_key.py @@ -81,6 +81,8 @@ class TDTestCase: finally: consumer.close() + time.sleep(4) # wait for heart beat + tdSql.query(f'show subscriptions;') sub = tdSql.getData(0, 4); print(sub) @@ -190,6 +192,7 @@ class TDTestCase: finally: consumer.close() + time.sleep(4) # wait for heart beat tdSql.query(f'show subscriptions;') sub = tdSql.getData(0, 4); print(sub) @@ -298,6 +301,7 @@ class TDTestCase: finally: consumer.close() + time.sleep(4) # wait for heart beat tdSql.query(f'show subscriptions;') sub = tdSql.getData(0, 4); print(sub) @@ -406,6 +410,7 @@ class TDTestCase: finally: consumer.close() + time.sleep(4) # wait for heart beat tdSql.query(f'show subscriptions;') sub = tdSql.getData(0, 4); print(sub) diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 6d99694df0..8e9a67eb41 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -1000,6 +1000,17 @@ void testConsumeExcluded(int topic_type){ } taos_free_result(pRes); } + +void testDetailError(){ + tmq_raw_data raw = {0}; + raw.raw_type = 2; + int32_t code = tmq_write_raw((TAOS *)1, raw); + ASSERT(code); + const char *err = tmq_err2str(code); + char* tmp = strstr(err, "Invalid parameters,detail:taos:0x1 or data"); + ASSERT(tmp != NULL); +} + int main(int argc, char* argv[]) { for (int32_t i = 1; i < argc; i++) { if (strcmp(argv[i], "-c") == 0) { @@ -1036,5 +1047,6 @@ int main(int argc, char* argv[]) { testConsumeExcluded(1); testConsumeExcluded(2); + testDetailError(); taosCloseFile(&g_fp); }