xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/compatibleWithVeryOldMeta

This commit is contained in:
Yihao Deng 2024-04-20 01:54:02 +00:00
parent 18cdeaf960 0acc5f485a
commit 517365467d
31 changed files with 1951 additions and 347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
contrib/CMakeLists.txt
View File

@ -109,6 +109,13 @@ cat("${TD_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
# cJson
cat("${TD_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
# xz
#cat("${TD_SUPPORT_DIR}/xz_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
#lzma2
cat("${TD_SUPPORT_DIR}/lzma_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
if (${BUILD_CONTRIB})
if(${BUILD_WITH_ROCKSDB})
cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@ -127,6 +134,8 @@ else()
endif()
endif()
#cat("${TD_SUPPORT_DIR}/zstd_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
#libuv
if(${BUILD_WITH_UV})
cat("${TD_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@ -254,6 +263,13 @@ target_include_directories(
)
unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# add_subdirectory(xz EXCLUDE_FROM_ALL)
# target_include_directories(
# xz
# PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/xz
# PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/xz
# )
# leveldb
if(${BUILD_WITH_LEVELDB})
option(LEVELDB_BUILD_TESTS "" OFF)
@ -388,7 +404,6 @@ endif()
if(${BUILD_WITH_S3})
INCLUDE_DIRECTORIES($ENV{HOME}/.cos-local.2/include)
MESSAGE("build with s3: ${BUILD_WITH_S3}")
# cos
elseif(${BUILD_WITH_COS})
if(${TD_LINUX})
@ -427,6 +442,7 @@ if(${BUILD_PTHREAD})
target_link_libraries(pthread INTERFACE libpthreadVC3)
endif()
# jemalloc
if(${JEMALLOC_ENABLED})
include(ExternalProject)

37
include/common/tmsg.h
View File

@ -3942,6 +3942,7 @@ int32_t tDecodeMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp);
void tDeleteMqMetaRsp(SMqMetaRsp *pRsp);
#define MQ_DATA_RSP_VERSION 100
typedef struct {
SMqRspHead head;
STqOffsetVal reqOffset;
@ -3953,33 +3954,27 @@ typedef struct {
SArray* blockData;
SArray* blockTbName;
SArray* blockSchema;
int64_t sleepTime;
} SMqDataRspCommon;
typedef struct {
SMqDataRspCommon common;
int64_t sleepTime;
} SMqDataRsp;
int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp);
int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp, int8_t dataVersion);
void tDeleteMqDataRsp(SMqDataRsp* pRsp);
int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const void* pRsp);
int32_t tDecodeMqDataRsp(SDecoder* pDecoder, void* pRsp);
void tDeleteMqDataRsp(void* pRsp);
typedef struct {
SMqRspHead head;
STqOffsetVal reqOffset;
STqOffsetVal rspOffset;
int32_t blockNum;
int8_t withTbName;
int8_t withSchema;
SArray* blockDataLen;
SArray* blockData;
SArray* blockTbName;
SArray* blockSchema;
// the following attributes are extended from SMqDataRsp
int32_t createTableNum;
SArray* createTableLen;
SArray* createTableReq;
SMqDataRspCommon common;
int32_t createTableNum;
SArray* createTableLen;
SArray* createTableReq;
} STaosxRsp;
int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const STaosxRsp* pRsp);
int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, STaosxRsp* pRsp, int8_t dateVersion);
void tDeleteSTaosxRsp(STaosxRsp* pRsp);
int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const void* pRsp);
int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, void* pRsp);
void tDeleteSTaosxRsp(void* pRsp);
typedef struct {
SMqRspHead head;

1
include/dnode/vnode/tqCommon.h
View File

@ -41,5 +41,6 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode);
void tqSetRestoreVersionInfo(SStreamTask* pTask);
#endif // TDENGINE_TQ_COMMON_H

3
include/libs/parser/parser.h
View File

@ -152,7 +152,8 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength);
int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD *fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen);
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);

8
include/util/taoserror.h
View File

@ -33,8 +33,16 @@ extern "C" {
const char* tstrerror(int32_t err);
const char* terrstr();
#define ERR_MSG_LEN 256
char* taosGetErrMsgReturn();
char* taosGetErrMsg();
int32_t* taosGetErrno();
#define terrno (*taosGetErrno())
#define terrMsg (taosGetErrMsg())
#define SET_ERROR_MSG(MSG, ...) \
snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__)
#define TSDB_CODE_SUCCESS 0
#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error

18
source/client/inc/clientInt.h
View File

@ -221,7 +221,11 @@ typedef struct {
SSchemaWrapper schema;
int32_t resIter;
SReqResultInfo resInfo;
SMqDataRsp rsp;
} SMqRspObjCommon;
typedef struct {
SMqRspObjCommon common;
SMqDataRsp rsp;
} SMqRspObj;
typedef struct {
@ -233,14 +237,8 @@ typedef struct {
} SMqMetaRspObj;
typedef struct {
int8_t resType;
char topic[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int32_t vgId;
SSchemaWrapper schema;
int32_t resIter;
SReqResultInfo resInfo;
STaosxRsp rsp;
SMqRspObjCommon common;
STaosxRsp rsp;
} SMqTaosxRspObj;
typedef struct SReqRelInfo {
@ -320,7 +318,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols);
static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
SMqRspObj* msg = (SMqRspObj*)res;
return (SReqResultInfo*)&msg->resInfo;
return (SReqResultInfo*)&msg->common.resInfo;
}
SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4);

6
source/client/src/clientMain.c
View File

@ -343,12 +343,12 @@ void taos_free_result(TAOS_RES *res) {
} else if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj *pRsp = (SMqTaosxRspObj *)res;
tDeleteSTaosxRsp(&pRsp->rsp);
doFreeReqResultInfo(&pRsp->resInfo);
doFreeReqResultInfo(&pRsp->common.resInfo);
taosMemoryFree(pRsp);
} else if (TD_RES_TMQ(res)) {
SMqRspObj *pRsp = (SMqRspObj *)res;
tDeleteMqDataRsp(&pRsp->rsp);
doFreeReqResultInfo(&pRsp->resInfo);
doFreeReqResultInfo(&pRsp->common.resInfo);
taosMemoryFree(pRsp);
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res;
@ -417,7 +417,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
} else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SMqRspObj *msg = ((SMqRspObj *)res);
SReqResultInfo *pResultInfo;
if (msg->resIter == -1) {
if (msg->common.resIter == -1) {
pResultInfo = tmqGetNextResInfo(res, true);
} else {
pResultInfo = tmqGetCurResInfo(res);

176
source/client/src/clientRawBlockWrite.c
View File

@ -1486,7 +1486,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData));
code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false);
code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
@ -1570,7 +1570,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData));
code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false);
code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
@ -1608,6 +1608,7 @@ static void* getRawDataFromRes(void* pRetrieve) {
static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
if (taos == NULL || data == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
return terrno;
}
int32_t code = TSDB_CODE_SUCCESS;
@ -1620,17 +1621,24 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
terrno = TSDB_CODE_SUCCESS;
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
if (!pRequest) {
SET_ERROR_MSG("pRequest is NULL");
return terrno;
}
uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
pRequest->syncQuery = true;
rspObj.resIter = -1;
rspObj.resType = RES_TYPE__TMQ;
rspObj.common.resIter = -1;
rspObj.common.resType = RES_TYPE__TMQ;
int8_t dataVersion = *(int8_t*)data;
if (dataVersion >= MQ_DATA_RSP_VERSION){
data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
dataLen -= sizeof(int8_t) + sizeof(int32_t);
}
tDecoderInit(&decoder, data, dataLen);
code = tDecodeMqDataRsp(&decoder, &rspObj.rsp, *(int8_t*)data);
code = tDecodeMqDataRsp(&decoder, &rspObj.rsp);
if (code != 0) {
SET_ERROR_MSG("decode mq data rsp failed");
code = TSDB_CODE_INVALID_MSG;
goto end;
}
@ -1643,6 +1651,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
struct SCatalog* pCatalog = NULL;
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("cata log get handle failed");
goto end;
}
@ -1655,17 +1664,19 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
pQuery = smlInitHandle();
if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
SET_ERROR_MSG("init sml handle failed");
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
while (++rspObj.resIter < rspObj.rsp.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter);
if (!rspObj.rsp.withSchema) {
while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter);
if (!rspObj.rsp.common.withSchema) {
goto end;
}
const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter);
const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter);
if (!tbName) {
SET_ERROR_MSG("block tbname is null");
code = TSDB_CODE_TMQ_INVALID_MSG;
goto end;
}
@ -1681,12 +1692,14 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
// continue;
// }
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("cata log get table:%s meta failed", tbName);
goto end;
}
SVgroupInfo vg;
code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("cata log get table:%s vgroup failed", tbName);
goto end;
}
@ -1695,9 +1708,11 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg));
}
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
if (fields == NULL) {
SET_ERROR_MSG("calloc fields failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
for (int i = 0; i < pSW->nCols; i++) {
@ -1706,16 +1721,18 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
}
void* rawData = getRawDataFromRes(pRetrieve);
code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true);
char err[ERR_MSG_LEN] = {0};
code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN);
taosMemoryFree(fields);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", tbName, err);
goto end;
}
taosMemoryFreeClear(pTableMeta);
}
code = smlBuildOutput(pQuery, pVgHash);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("sml build output failed");
goto end;
}
@ -1737,6 +1754,7 @@ end:
static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
if (taos == NULL || data == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
return terrno;
}
int32_t code = TSDB_CODE_SUCCESS;
@ -1750,16 +1768,24 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
terrno = TSDB_CODE_SUCCESS;
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
if (!pRequest) {
SET_ERROR_MSG("pRequest is NULL");
return terrno;
}
uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
pRequest->syncQuery = true;
rspObj.resIter = -1;
rspObj.resType = RES_TYPE__TMQ_METADATA;
rspObj.common.resIter = -1;
rspObj.common.resType = RES_TYPE__TMQ_METADATA;
int8_t dataVersion = *(int8_t*)data;
if (dataVersion >= MQ_DATA_RSP_VERSION){
data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
dataLen -= sizeof(int8_t) + sizeof(int32_t);
}
tDecoderInit(&decoder, data, dataLen);
code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp, *(int8_t*)data);
code = tDecodeSTaosxRsp(&decoder, &rspObj.rsp);
if (code != 0) {
SET_ERROR_MSG("decode mq taosx data rsp failed");
code = TSDB_CODE_INVALID_MSG;
goto end;
}
@ -1772,6 +1798,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
struct SCatalog* pCatalog = NULL;
code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("cata log get handle failed");
goto end;
}
@ -1783,20 +1810,22 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
pQuery = smlInitHandle();
if (pQuery == NULL) {
SET_ERROR_MSG("init sml handle failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.blockNum);
while (++rspObj.resIter < rspObj.rsp.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter);
if (!rspObj.rsp.withSchema) {
uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum);
while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter);
if (!rspObj.rsp.common.withSchema) {
goto end;
}
const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter);
const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.common.blockTbName, rspObj.common.resIter);
if (!tbName) {
SET_ERROR_MSG("block tbname is null");
code = TSDB_CODE_TMQ_INVALID_MSG;
goto end;
}
@ -1818,6 +1847,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
tDecoderClear(&decoderTmp);
tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
code = TSDB_CODE_TMQ_INVALID_MSG;
SET_ERROR_MSG("decode create table:%s req failed", tbName);
goto end;
}
@ -1825,6 +1855,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
code = TSDB_CODE_TSC_INVALID_VALUE;
tDecoderClear(&decoderTmp);
tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE);
SET_ERROR_MSG("create table req type is not child table: %s, type: %d", tbName, pCreateReq.type);
goto end;
}
if (strcmp(tbName, pCreateReq.name) == 0) {
@ -1841,6 +1872,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
SVgroupInfo vg;
code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("cata log get table:%s vgroup failed", tbName);
goto end;
}
@ -1854,6 +1886,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
// continue;
// }
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("cata log get table:%s meta failed", tbName);
goto end;
}
@ -1867,9 +1900,11 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg));
}
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.common.blockSchema, rspObj.common.resIter);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
if (fields == NULL) {
SET_ERROR_MSG("calloc fields failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
for (int i = 0; i < pSW->nCols; i++) {
@ -1878,12 +1913,13 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
}
void* rawData = getRawDataFromRes(pRetrieve);
code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true);
char err[ERR_MSG_LEN] = {0};
code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN);
taosMemoryFree(fields);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", tbName, err);
goto end;
}
taosMemoryFreeClear(pTableMeta);
}
code = smlBuildOutput(pQuery, pVgHash);
@ -1946,6 +1982,64 @@ char* tmq_get_json_meta(TAOS_RES* res) {
void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); }
static int32_t getOffSetLen(const void *rsp){
const SMqDataRspCommon *pRsp = rsp;
SEncoder coder = {0};
tEncoderInit(&coder, NULL, 0);
if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1;
if (tEncodeSTqOffsetVal(&coder, &pRsp->rspOffset) < 0) return -1;
int32_t pos = coder.pos;
tEncoderClear(&coder);
return pos;
}
typedef int32_t __encode_func__(SEncoder *pEncoder, const void *pRsp);
static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, void* rspObj, tmq_raw_data* raw){
int32_t len = 0;
int32_t code = 0;
SEncoder encoder = {0};
void* buf = NULL;
tEncodeSize(encodeFunc, rspObj, len, code);
if (code < 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
len += sizeof(int8_t) + sizeof(int32_t);
buf = taosMemoryCalloc(1, len);
if(buf == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAILED;
}
tEncoderInit(&encoder, buf, len);
if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
int32_t offsetLen = getOffSetLen(rspObj);
if(offsetLen <= 0){
terrno = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
if (tEncodeI32(&encoder, offsetLen) < 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
if(encodeFunc(&encoder, rspObj) < 0){
terrno = TSDB_CODE_INVALID_MSG;
goto FAILED;
}
tEncoderClear(&encoder);
raw->raw = buf;
raw->raw_len = len;
return 0;
FAILED:
tEncoderClear(&encoder);
taosMemoryFree(buf);
return terrno;
}
int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
if (!raw || !res) {
terrno = TSDB_CODE_INVALID_PARA;
@ -1959,42 +2053,19 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
uDebug("tmq get raw type meta:%p", raw);
} else if (TD_RES_TMQ(res)) {
SMqRspObj* rspObj = ((SMqRspObj*)res);
int32_t len = 0;
int32_t code = 0;
tEncodeSize(tEncodeMqDataRsp, &rspObj->rsp, len, code);
if (code < 0) {
return -1;
if(encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->rsp, raw) != 0){
uError("tmq get raw type error:%d", terrno);
return terrno;
}
void* buf = taosMemoryCalloc(1, len);
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, len);
tEncodeMqDataRsp(&encoder, &rspObj->rsp);
tEncoderClear(&encoder);
raw->raw = buf;
raw->raw_len = len;
raw->raw_type = RES_TYPE__TMQ;
uDebug("tmq get raw type data:%p", raw);
} else if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res);
int32_t len = 0;
int32_t code = 0;
tEncodeSize(tEncodeSTaosxRsp, &rspObj->rsp, len, code);
if (code < 0) {
return -1;
if(encodeMqDataRsp(tEncodeSTaosxRsp, &rspObj->rsp, raw) != 0){
uError("tmq get raw type error:%d", terrno);
return terrno;
}
void* buf = taosMemoryCalloc(1, len);
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, len);
tEncodeSTaosxRsp(&encoder, &rspObj->rsp);
tEncoderClear(&encoder);
raw->raw = buf;
raw->raw_len = len;
raw->raw_type = RES_TYPE__TMQ_METADATA;
uDebug("tmq get raw type metadata:%p", raw);
} else {
@ -2010,6 +2081,7 @@ void tmq_free_raw(tmq_raw_data raw) {
if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) {
taosMemoryFree(raw.raw);
}
memset(terrMsg, 0, ERR_MSG_LEN);
}
int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {

172
source/client/src/clientTmq.c
View File

@ -622,9 +622,9 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c
if (TD_RES_TMQ(pRes)) {
SMqRspObj* pRspObj = (SMqRspObj*)pRes;
pTopicName = pRspObj->topic;
vgId = pRspObj->vgId;
offsetVal = pRspObj->rsp.rspOffset;
pTopicName = pRspObj->common.topic;
vgId = pRspObj->common.vgId;
offsetVal = pRspObj->rsp.common.rspOffset;
} else if (TD_RES_TMQ_META(pRes)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes;
pTopicName = pMetaRspObj->topic;
@ -632,9 +632,9 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c
offsetVal = pMetaRspObj->metaRsp.rspOffset;
} else if (TD_RES_TMQ_METADATA(pRes)) {
SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes;
pTopicName = pRspObj->topic;
vgId = pRspObj->vgId;
offsetVal = pRspObj->rsp.rspOffset;
pTopicName = pRspObj->common.topic;
vgId = pRspObj->common.vgId;
offsetVal = pRspObj->rsp.common.rspOffset;
} else {
code = TSDB_CODE_TMQ_INVALID_MSG;
goto end;
@ -1056,17 +1056,17 @@ static void tmqMgmtInit(void) {
}
}
#define SET_ERROR_MSG(MSG) \
#define SET_ERROR_MSG_TMQ(MSG) \
if (errstr != NULL) snprintf(errstr, errstrLen, MSG);
tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
if (conf == NULL) {
SET_ERROR_MSG("configure is null")
SET_ERROR_MSG_TMQ("configure is null")
return NULL;
}
taosThreadOnce(&tmqInit, tmqMgmtInit);
if (tmqInitRes != 0) {
terrno = tmqInitRes;
SET_ERROR_MSG("tmq timer init error")
SET_ERROR_MSG_TMQ("tmq timer init error")
return NULL;
}
@ -1074,7 +1074,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
if (pTmq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tscError("failed to create consumer, groupId:%s, code:%s", conf->groupId, terrstr());
SET_ERROR_MSG("malloc tmq failed")
SET_ERROR_MSG_TMQ("malloc tmq failed")
return NULL;
}
@ -1090,7 +1090,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
conf->groupId[0] == 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
SET_ERROR_MSG("malloc tmq element failed or group is empty")
SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty")
goto _failed;
}
@ -1123,7 +1123,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
if (tsem_init(&pTmq->rspSem, 0, 0) != 0) {
tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(),
pTmq->groupId);
SET_ERROR_MSG("init t_sem failed")
SET_ERROR_MSG_TMQ("init t_sem failed")
goto _failed;
}
@ -1132,13 +1132,13 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
if (pTmq->pTscObj == NULL) {
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
tsem_destroy(&pTmq->rspSem);
SET_ERROR_MSG("init tscObj failed")
SET_ERROR_MSG_TMQ("init tscObj failed")
goto _failed;
}
pTmq->refId = taosAddRef(tmqMgmt.rsetId, pTmq);
if (pTmq->refId < 0) {
SET_ERROR_MSG("add tscObj ref failed")
SET_ERROR_MSG_TMQ("add tscObj ref failed")
goto _failed;
}
@ -1377,7 +1377,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) {
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp, *(int8_t*)POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead))) < 0){
if(tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0){
tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1387,9 +1387,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.rspOffset);
tFormatOffset(buf, TSDB_OFFSET_LEN, &pRspWrapper->dataRsp.common.rspOffset);
tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64,
tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId);
tmq->consumerId, vgId, pRspWrapper->dataRsp.common.reqOffset.version, buf, rspType, requestId);
} else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
@ -1404,7 +1404,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
} else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
if(tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp, *(int8_t*)POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead))) < 0){
if(tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0){
tDecoderClear(&decoder);
taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY;
@ -1598,6 +1598,9 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl
SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj));
if(pRspObj == NULL) {
return NULL;
}
pRspObj->resType = RES_TYPE__TMQ_META;
tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
@ -1611,8 +1614,7 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
void changeByteEndian(char* pData){
char* p = pData;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length |
// version:
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | version:
int32_t blockVersion = *(int32_t*)p;
ASSERT(blockVersion == BLOCK_VERSION_1);
*(int32_t*)p = BLOCK_VERSION_2;
@ -1649,7 +1651,7 @@ static void tmqGetRawDataRowsPrecisionFromRes(void *pRetrieve, void** rawData, i
}
}
static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) {
static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) {
(*numOfRows) = 0;
tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
@ -1660,14 +1662,14 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
pRspObj->resInfo.totalRows = 0;
pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
bool needTransformSchema = !pRspObj->rsp.withSchema;
if (!pRspObj->rsp.withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
pRspObj->rsp.withSchema = true;
pRspObj->rsp.blockSchema = taosArrayInit(pRspObj->rsp.blockNum, sizeof(void*));
bool needTransformSchema = !pDataRsp->withSchema;
if (!pDataRsp->withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable
pDataRsp->withSchema = true;
pDataRsp->blockSchema = taosArrayInit(pDataRsp->blockNum, sizeof(void*));
}
// extract the rows in this data packet
for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i);
for (int32_t i = 0; i < pDataRsp->blockNum; ++i) {
void* pRetrieve = taosArrayGetP(pDataRsp->blockData, i);
void* rawData = NULL;
int64_t rows = 0;
// deal with compatibility
@ -1679,7 +1681,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable
SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
if(schema){
taosArrayPush(pRspObj->rsp.blockSchema, &schema);
taosArrayPush(pDataRsp->blockSchema, &schema);
}
}
}
@ -1687,18 +1689,24 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg
SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
pRspObj->resType = RES_TYPE__TMQ;
if(pRspObj == NULL){
return NULL;
}
pRspObj->common.resType = RES_TYPE__TMQ;
memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj);
tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common);
return pRspObj;
}
SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
pRspObj->resType = RES_TYPE__TMQ_METADATA;
if(pRspObj == NULL){
return NULL;
}
pRspObj->common.resType = RES_TYPE__TMQ_METADATA;
memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp));
tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj);
tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, &pRspObj->common, &pRspObj->rsp.common);
return pRspObj;
}
@ -1891,7 +1899,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp;
SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->dataRsp;
if (pDataRsp->head.epoch == consumerEpoch) {
taosWLockLatch(&tmq->lock);
@ -1994,8 +2002,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
} else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp;
if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) {
if (pDataRsp->head.epoch == consumerEpoch) {
taosWLockLatch(&tmq->lock);
SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
pollRspWrapper->vgHandle = pVg;
@ -2009,11 +2018,11 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
return NULL;
}
updateVgInfo(pVg, &pollRspWrapper->taosxRsp.reqOffset, &pollRspWrapper->taosxRsp.rspOffset,
pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId,
pollRspWrapper->taosxRsp.blockNum != 0);
updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset,
pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId,
pDataRsp->blockNum != 0);
if (pollRspWrapper->taosxRsp.blockNum == 0) {
if (pDataRsp->blockNum == 0) {
tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64,
tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId);
pVg->emptyBlockReceiveTs = taosGetTimestampMs();
@ -2034,7 +2043,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset);
tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows,
tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows,
tmq->totalRows, pollRspWrapper->reqId);
taosFreeQitem(pRspWrapper);
@ -2042,7 +2051,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
return pRsp;
} else {
tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch);
tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch);
setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId);
tmqFreeRspWrapper(pRspWrapper);
taosFreeQitem(pRspWrapper);
@ -2182,7 +2191,12 @@ const char* tmq_err2str(int32_t err) {
} else if (err == -1) {
return "fail";
} else {
return tstrerror(err);
if(*(taosGetErrMsg()) == 0){
return tstrerror(err);
} else{
snprintf(taosGetErrMsgReturn(), ERR_MSG_LEN, "%s,detail:%s", tstrerror(err), taosGetErrMsg());
return (const char*)taosGetErrMsgReturn();
}
}
}
@ -2205,15 +2219,11 @@ const char* tmq_get_topic_name(TAOS_RES* res) {
if (res == NULL) {
return NULL;
}
if (TD_RES_TMQ(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res;
return strchr(pRspObj->topic, '.') + 1;
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
return strchr(((SMqRspObjCommon*)res)->topic, '.') + 1;
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
return strchr(pMetaRspObj->topic, '.') + 1;
} else if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
return strchr(pRspObj->topic, '.') + 1;
} else {
return NULL;
}
@ -2224,15 +2234,11 @@ const char* tmq_get_db_name(TAOS_RES* res) {
return NULL;
}
if (TD_RES_TMQ(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res;
return strchr(pRspObj->db, '.') + 1;
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
return strchr(((SMqRspObjCommon*)res)->db, '.') + 1;
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
return strchr(pMetaRspObj->db, '.') + 1;
} else if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
return strchr(pRspObj->db, '.') + 1;
} else {
return NULL;
}
@ -2242,15 +2248,11 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
if (res == NULL) {
return -1;
}
if (TD_RES_TMQ(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res;
return pRspObj->vgId;
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
return ((SMqRspObjCommon*)res)->vgId;
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
return pMetaRspObj->vgId;
} else if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
return pRspObj->vgId;
} else {
return -1;
}
@ -2260,24 +2262,19 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) {
if (res == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (TD_RES_TMQ(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res;
STqOffsetVal* pOffset = &pRspObj->rsp.reqOffset;
if (pOffset->type == TMQ_OFFSET__LOG) {
return pRspObj->rsp.reqOffset.version;
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
STqOffsetVal* pOffset = &common->reqOffset;
if (common->reqOffset.type == TMQ_OFFSET__LOG) {
return common->reqOffset.version;
} else {
tscError("invalid offset type:%d", pOffset->type);
tscError("invalid offset type:%d", common->reqOffset.type);
}
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pRspObj = (SMqMetaRspObj*)res;
if (pRspObj->metaRsp.rspOffset.type == TMQ_OFFSET__LOG) {
return pRspObj->metaRsp.rspOffset.version;
}
} else if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
if (pRspObj->rsp.reqOffset.type == TMQ_OFFSET__LOG) {
return pRspObj->rsp.reqOffset.version;
}
} else {
tscError("invalid tmq type:%d", *(int8_t*)res);
}
@ -2290,20 +2287,15 @@ const char* tmq_get_table_name(TAOS_RES* res) {
if (res == NULL) {
return NULL;
}
if (TD_RES_TMQ(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res;
if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 ||
pRspObj->resIter >= pRspObj->rsp.blockNum) {
if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res;
if (!common->withTbName || common->blockTbName == NULL || pRspObj->resIter < 0 ||
pRspObj->resIter >= common->blockNum) {
return NULL;
}
return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter);
} else if (TD_RES_TMQ_METADATA(res)) {
SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res;
if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 ||
pRspObj->resIter >= pRspObj->rsp.blockNum) {
return NULL;
}
return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter);
return (const char*)taosArrayGetP(common->blockTbName, pRspObj->resIter);
}
return NULL;
}
@ -2640,17 +2632,17 @@ void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, cons
}
SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
SMqRspObj* pRspObj = (SMqRspObj*)res;
SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon));
SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res;
pRspObj->resIter++;
if (pRspObj->resIter < pRspObj->rsp.blockNum) {
if (pRspObj->rsp.withSchema) {
if (pRspObj->resIter < common->blockNum) {
if (common->withSchema) {
doFreeReqResultInfo(&pRspObj->resInfo);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(common->blockSchema, pRspObj->resIter);
setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
}
void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter);
void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter);
void* rawData = NULL;
int64_t rows = 0;
int32_t precision = 0;
@ -2685,14 +2677,14 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
SMqDataRsp rsp;
SDecoder decoder;
tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
tDecodeMqDataRsp(&decoder, &rsp, *(int8_t*)POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)));
tDecodeMqDataRsp(&decoder, &rsp);
tDecoderClear(&decoder);
SMqRspHead* pHead = pMsg->pData;
tmq_topic_assignment assignment = {.begin = pHead->walsver,
.end = pHead->walever + 1,
.currentOffset = rsp.rspOffset.version,
.currentOffset = rsp.common.rspOffset.version,
.vgId = pParam->vgId};
taosThreadMutexLock(&pCommon->mutex);

54
source/common/src/tmsg.c
View File

@ -9173,7 +9173,7 @@ int32_t tDecodeMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) {
void tDeleteMqMetaRsp(SMqMetaRsp *pRsp) { taosMemoryFree(pRsp->metaRsp); }
int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRspCommon *pRsp) {
if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1;
if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1;
@ -9198,14 +9198,13 @@ int32_t tEncodeMqDataRspCommon(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
return 0;
}
int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
if (tEncodeI8(pEncoder, MQ_DATA_RSP_VERSION) < 0) return -1;
int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const void *pRsp) {
if (tEncodeMqDataRspCommon(pEncoder, pRsp) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->sleepTime) < 0) return -1;
if (tEncodeI64(pEncoder, ((SMqDataRsp*)pRsp)->sleepTime) < 0) return -1;
return 0;
}
int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) {
int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRspCommon *pRsp) {
if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1;
if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1;
@ -9251,19 +9250,17 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) {
return 0;
}
int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp, int8_t dataVersion) {
if (dataVersion >= MQ_DATA_RSP_VERSION) {
if (tDecodeI8(pDecoder, &dataVersion) < 0) return -1;
}
int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) {
if (tDecodeMqDataRspCommon(pDecoder, pRsp) < 0) return -1;
if (!tDecodeIsEnd(pDecoder)) {
if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1;
if (tDecodeI64(pDecoder, &((SMqDataRsp*)pRsp)->sleepTime) < 0) return -1;
}
return 0;
}
void tDeleteMqDataRsp(SMqDataRsp *pRsp) {
static void tDeleteMqDataRspCommon(void *rsp) {
SMqDataRspCommon *pRsp = rsp;
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
pRsp->blockData = NULL;
@ -9275,10 +9272,14 @@ void tDeleteMqDataRsp(SMqDataRsp *pRsp) {
tOffsetDestroy(&pRsp->rspOffset);
}
int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
if (tEncodeI8(pEncoder, MQ_DATA_RSP_VERSION) < 0) return -1;
if (tEncodeMqDataRspCommon(pEncoder, (const SMqDataRsp *)pRsp) < 0) return -1;
void tDeleteMqDataRsp(void *rsp) {
tDeleteMqDataRspCommon(rsp);
}
int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const void *rsp) {
if (tEncodeMqDataRspCommon(pEncoder, rsp) < 0) return -1;
const STaosxRsp *pRsp = (const STaosxRsp *)rsp;
if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1;
if (pRsp->createTableNum) {
for (int32_t i = 0; i < pRsp->createTableNum; i++) {
@ -9290,19 +9291,17 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
return 0;
}
int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp, int8_t dataVersion) {
if (dataVersion >= MQ_DATA_RSP_VERSION) {
if (tDecodeI8(pDecoder, &dataVersion) < 0) return -1;
}
if (tDecodeMqDataRspCommon(pDecoder, (SMqDataRsp *)pRsp) < 0) return -1;
int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, void *rsp) {
if (tDecodeMqDataRspCommon(pDecoder, rsp) < 0) return -1;
STaosxRsp *pRsp = (STaosxRsp *)rsp;
if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1;
if (pRsp->createTableNum) {
pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
for (int32_t i = 0; i < pRsp->createTableNum; i++) {
void *pCreate = NULL;
uint64_t len;
void * pCreate = NULL;
uint64_t len = 0;
if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
int32_t l = (int32_t)len;
taosArrayPush(pRsp->createTableLen, &l);
@ -9313,20 +9312,13 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp, int8_t dataVersion
return 0;
}
void tDeleteSTaosxRsp(STaosxRsp *pRsp) {
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
pRsp->blockData = NULL;
taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper);
pRsp->blockSchema = NULL;
taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
pRsp->blockTbName = NULL;
void tDeleteSTaosxRsp(void *rsp) {
tDeleteMqDataRspCommon(rsp);
STaosxRsp *pRsp = (STaosxRsp *)rsp;
pRsp->createTableLen = taosArrayDestroy(pRsp->createTableLen);
taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree);
pRsp->createTableReq = NULL;
tOffsetDestroy(&pRsp->reqOffset);
tOffsetDestroy(&pRsp->rspOffset);
}
int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) {

1
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -648,7 +648,6 @@ static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscri
_over:
mndReleaseConsumer(pMnode, pExistedConsumer);
tDeleteSMqConsumerObj(pConsumerNew);
taosArrayDestroyP(subscribe->topicNames, (FDelete)taosMemoryFree);
return NULL;
}

12
source/dnode/snode/src/snode.c
View File

@ -58,17 +58,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
streamSetupScheduleTrigger(pTask);
SCheckpointInfo *pChkInfo = &pTask->chkInfo;
// checkpoint ver is the kept version, handled data should be the next version.
if (pChkInfo->checkpointId != 0) {
pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1;
pChkInfo->processedVer = pChkInfo->checkpointVer;
pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer;
pTask->execInfo.startCheckpointId = pChkInfo->checkpointId;
sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64,
pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
}
tqSetRestoreVersionInfo(pTask);
char* p = streamTaskGetStatus(pTask)->name;
if (pTask->info.fillHistory) {

8
source/dnode/vnode/src/inc/tq.h
View File

@ -119,8 +119,8 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t
// tqExec
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded);
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision);
int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp,
int32_t type, int32_t vgId);
int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);
@ -154,9 +154,9 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
// tq util
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type);
int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg);
int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId,
int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId,
int32_t type, int64_t sver, int64_t ever);
int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset);
int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset);
void tqUpdateNodeStage(STQ* pTq, bool isLeader);
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData, const char* id);

41
source/dnode/vnode/src/tq/tq.c
View File

@ -153,10 +153,10 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) {
}
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, req.reqOffset);
dataRsp.blockNum = 0;
tqInitDataRsp(&dataRsp.common, req.reqOffset);
dataRsp.common.blockNum = 0;
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.reqOffset);
tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.reqOffset);
tqInfo("tqPushEmptyDataRsp to consumer:0x%" PRIx64 " vgId:%d, offset:%s, reqId:0x%" PRIx64, req.consumerId, vgId, buf,
req.reqId);
@ -165,7 +165,7 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) {
return 0;
}
int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const void* pRsp,
int32_t type, int32_t vgId) {
int64_t sver = 0, ever = 0;
walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever);
@ -174,11 +174,11 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq*
char buf1[TSDB_OFFSET_LEN] = {0};
char buf2[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf1, TSDB_OFFSET_LEN, &pRsp->reqOffset);
tFormatOffset(buf2, TSDB_OFFSET_LEN, &pRsp->rspOffset);
tFormatOffset(buf1, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->reqOffset);
tFormatOffset(buf2, TSDB_OFFSET_LEN, &((SMqDataRspCommon*)pRsp)->rspOffset);
tqDebug("tmq poll vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64,
vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId);
vgId, pReq->consumerId, pReq->epoch, ((SMqDataRspCommon*)pRsp)->blockNum, buf1, buf2, pReq->reqId);
return 0;
}
@ -499,7 +499,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
taosRUnLockLatch(&pTq->lock);
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, req.reqOffset);
tqInitDataRsp(&dataRsp.common, req.reqOffset);
if (req.useSnapshot == true) {
tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey);
@ -508,10 +508,10 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
return -1;
}
dataRsp.rspOffset.type = TMQ_OFFSET__LOG;
dataRsp.common.rspOffset.type = TMQ_OFFSET__LOG;
if (reqOffset.type == TMQ_OFFSET__LOG) {
dataRsp.rspOffset.version = reqOffset.version;
dataRsp.common.rspOffset.version = reqOffset.version;
} else if (reqOffset.type < 0) {
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey);
if (pOffset != NULL) {
@ -522,17 +522,17 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
return -1;
}
dataRsp.rspOffset.version = pOffset->val.version;
dataRsp.common.rspOffset.version = pOffset->val.version;
tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from store:%" PRId64, consumerId, vgId,
req.subKey, dataRsp.rspOffset.version);
req.subKey, dataRsp.common.rspOffset.version);
} else {
if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEST) {
dataRsp.rspOffset.version = sver; // not consume yet, set the earliest position
dataRsp.common.rspOffset.version = sver; // not consume yet, set the earliest position
} else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
dataRsp.rspOffset.version = ever;
dataRsp.common.rspOffset.version = ever;
}
tqInfo("consumer:0x%" PRIx64 " vgId:%d subkey:%s get assignment from init:%" PRId64, consumerId, vgId, req.subKey,
dataRsp.rspOffset.version);
dataRsp.common.rspOffset.version);
}
} else {
tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey,
@ -760,16 +760,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
streamSetupScheduleTrigger(pTask);
SCheckpointInfo* pChkInfo = &pTask->chkInfo;
// checkpoint ver is the kept version, handled data should be the next version.
if (pChkInfo->checkpointId != 0) {
pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1;
pChkInfo->processedVer = pChkInfo->checkpointVer;
pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer;
pTask->execInfo.startCheckpointId = pChkInfo->checkpointId;
tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr,
pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
}
tqSetRestoreVersionInfo(pTask);
char* p = streamTaskGetStatus(pTask)->name;
const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus);

50
source/dnode/vnode/src/tq/tqScan.c
View File

@ -15,7 +15,7 @@
#include "tq.h"
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) {
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, void* pRsp, int32_t numOfCols, int8_t precision) {
int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + blockGetEncodeSize(pBlock);
void* buf = taosMemoryCalloc(1, dataStrLen);
if (buf == NULL) {
@ -30,22 +30,22 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t
int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);
actualLen += sizeof(SRetrieveTableRspForTmq);
taosArrayPush(pRsp->blockDataLen, &actualLen);
taosArrayPush(pRsp->blockData, &buf);
taosArrayPush(((SMqDataRspCommon*)pRsp)->blockDataLen, &actualLen);
taosArrayPush(((SMqDataRspCommon*)pRsp)->blockData, &buf);
return TSDB_CODE_SUCCESS;
}
static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) {
static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, void* pRsp) {
SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper);
if (pSW == NULL) {
return -1;
}
taosArrayPush(pRsp->blockSchema, &pSW);
taosArrayPush(((SMqDataRspCommon*)pRsp)->blockSchema, &pSW);
return 0;
}
static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) {
static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, void* pRsp, int32_t n) {
SMetaReader mr = {0};
metaReaderDoInit(&mr, pTq->pVnode->pMeta, META_READER_LOCK);
@ -57,7 +57,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in
for (int32_t i = 0; i < n; i++) {
char* tbName = taosStrdup(mr.me.name);
taosArrayPush(pRsp->blockTbName, &tbName);
taosArrayPush(((SMqDataRspCommon*)pRsp)->blockTbName, &tbName);
}
metaReaderClear(&mr);
return 0;
@ -125,7 +125,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
return code;
}
pRsp->blockNum++;
pRsp->common.blockNum++;
if (pDataBlock == NULL) {
blockDataDestroy(pHandle->block);
pHandle->block = NULL;
@ -149,7 +149,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
return code;
}
pRsp->blockNum++;
pRsp->common.blockNum++;
totalRows += pDataBlock->info.rows;
if (totalRows >= tmqRowSize) {
break;
@ -158,8 +158,8 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
}
tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d",
pHandle->consumerId, vgId, pRsp->blockNum, totalRows);
qStreamExtractOffset(task, &pRsp->rspOffset);
pHandle->consumerId, vgId, pRsp->common.blockNum, totalRows);
qStreamExtractOffset(task, &pRsp->common.rspOffset);
return 0;
}
@ -186,7 +186,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
tqDebug("tmqsnap task execute end, get %p", pDataBlock);
if (pDataBlock != NULL && pDataBlock->info.rows > 0) {
if (pRsp->withTbName) {
if (pRsp->common.withTbName) {
if (pOffset->type == TMQ_OFFSET__LOG) {
int64_t uid = pExec->pTqReader->lastBlkUid;
if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) {
@ -194,21 +194,21 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
}
} else {
char* tbName = taosStrdup(qExtractTbnameFromTask(task));
taosArrayPush(pRsp->blockTbName, &tbName);
taosArrayPush(pRsp->common.blockTbName, &tbName);
}
}
if (pRsp->withSchema) {
if (pRsp->common.withSchema) {
if (pOffset->type == TMQ_OFFSET__LOG) {
tqAddBlockSchemaToRsp(pExec, pRsp);
} else {
SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
taosArrayPush(pRsp->blockSchema, &pSW);
taosArrayPush(pRsp->common.blockSchema, &pSW);
}
}
tqAddBlockDataToRsp(pDataBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pDataBlock->pDataBlock),
pTq->pVnode->config.tsdbCfg.precision);
pRsp->blockNum++;
pRsp->common.blockNum++;
if (pOffset->type == TMQ_OFFSET__LOG) {
continue;
} else {
@ -234,13 +234,13 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
}
tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode),
pHandle->snapshotVer + 1);
qStreamExtractOffset(task, &pRsp->rspOffset);
qStreamExtractOffset(task, &pRsp->common.rspOffset);
break;
}
if (pRsp->blockNum > 0) {
if (pRsp->common.blockNum > 0) {
tqDebug("tmqsnap task exec exited, get data");
qStreamExtractOffset(task, &pRsp->rspOffset);
qStreamExtractOffset(task, &pRsp->common.rspOffset);
break;
}
}
@ -268,7 +268,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) {
goto loop_table;
}
if (pRsp->withTbName) {
if (pRsp->common.withTbName) {
int64_t uid = pExec->pTqReader->lastBlkUid;
if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
goto loop_table;
@ -312,8 +312,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
*totalRows += pBlock->info.rows;
blockDataFreeRes(pBlock);
SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i);
taosArrayPush(pRsp->blockSchema, &pSW);
pRsp->blockNum++;
taosArrayPush(pRsp->common.blockSchema, &pSW);
pRsp->common.blockNum++;
}
continue;
loop_table:
@ -336,7 +336,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) {
goto loop_db;
}
if (pRsp->withTbName) {
if (pRsp->common.withTbName) {
int64_t uid = pExec->pTqReader->lastBlkUid;
if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
goto loop_db;
@ -380,8 +380,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
*totalRows += pBlock->info.rows;
blockDataFreeRes(pBlock);
SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i);
taosArrayPush(pRsp->blockSchema, &pSW);
pRsp->blockNum++;
taosArrayPush(pRsp->common.blockSchema, &pSW);
pRsp->common.blockNum++;
}
continue;
loop_db:

52
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -18,7 +18,7 @@
static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq,
const SMqMetaRsp* pRsp, int32_t vgId);
int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) {
tOffsetCopy(&pRsp->reqOffset, &pOffset);
tOffsetCopy(&pRsp->rspOffset, &pOffset);
@ -39,7 +39,7 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) {
streamMetaUpdateStageRole(pTq->pStreamMeta, state.term, isLeader);
}
static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) {
static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) {
tOffsetCopy(&pRsp->reqOffset, &pOffset);
tOffsetCopy(&pRsp->rspOffset, &pOffset);
@ -110,9 +110,9 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
SMqDataRsp dataRsp = {0};
tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer + 1);
tqInitDataRsp(&dataRsp, *pOffsetVal);
tqInitDataRsp(&dataRsp.common, *pOffsetVal);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId,
pHandle->subKey, vgId, dataRsp.rspOffset.version);
pHandle->subKey, vgId, dataRsp.common.rspOffset.version);
int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
tDeleteMqDataRsp(&dataRsp);
@ -137,7 +137,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
terrno = 0;
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, *pOffset);
tqInitDataRsp(&dataRsp.common, *pOffset);
qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
int code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest);
@ -146,11 +146,11 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
}
// till now, all data has been transferred to consumer, new data needs to push client once arrived.
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) {
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.common.blockNum == 0) {
// lock
taosWLockLatch(&pTq->lock);
int64_t ver = walGetCommittedVer(pTq->pVnode->pWal);
if (dataRsp.rspOffset.version > ver) { // check if there are data again to avoid lost data
if (dataRsp.common.rspOffset.version > ver) { // check if there are data again to avoid lost data
code = tqRegisterPushHandle(pTq, pHandle, pMsg);
taosWUnLockLatch(&pTq->lock);
goto end;
@ -158,15 +158,15 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
taosWUnLockLatch(&pTq->lock);
}
tOffsetCopy(&dataRsp.reqOffset, pOffset); // reqOffset represents the current date offset, may be changed if wal not exists
code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
tOffsetCopy(&dataRsp.common.reqOffset, pOffset); // reqOffset represents the current date offset, may be changed if wal not exists
code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
end : {
char buf[TSDB_OFFSET_LEN] = {0};
tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.rspOffset);
tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.common.rspOffset);
tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64
" code:%d",
consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code);
consumerId, pHandle->subKey, vgId, dataRsp.common.blockNum, buf, pRequest->reqId, code);
tDeleteMqDataRsp(&dataRsp);
return code;
}
@ -194,7 +194,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
int32_t vgId = TD_VID(pTq->pVnode);
SMqMetaRsp metaRsp = {0};
STaosxRsp taosxRsp = {0};
tqInitTaosxRsp(&taosxRsp, *offset);
tqInitTaosxRsp(&taosxRsp.common, *offset);
if (offset->type != TMQ_OFFSET__LOG) {
if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) {
@ -214,13 +214,13 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64
",ts:%" PRId64,
pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type,
taosxRsp.rspOffset.uid, taosxRsp.rspOffset.ts);
if (taosxRsp.blockNum > 0) {
code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.common.blockNum, taosxRsp.common.rspOffset.type,
taosxRsp.common.rspOffset.uid, taosxRsp.common.rspOffset.ts);
if (taosxRsp.common.blockNum > 0) {
code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
goto end;
} else {
tOffsetCopy(offset, &taosxRsp.rspOffset);
tOffsetCopy(offset, &taosxRsp.common.rspOffset);
}
}
@ -235,9 +235,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
ASSERT(savedEpoch <= pRequest->epoch);
if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) {
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer);
code = tqSendDataRsp(
pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp,
pHandle, pMsg, pRequest, &taosxRsp,
taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
goto end;
}
@ -249,9 +249,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
// process meta
if (pHead->msgType != TDMT_VND_SUBMIT) {
if (totalRows > 0) {
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer);
code = tqSendDataRsp(
pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp,
pHandle, pMsg, pRequest, &taosxRsp,
taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
goto end;
}
@ -292,9 +292,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
}
if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) {
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1);
tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer + 1);
code = tqSendDataRsp(
pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp,
pHandle, pMsg, pRequest, &taosxRsp,
taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
goto end;
} else {
@ -386,7 +386,7 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll
return 0;
}
int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId,
int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t epoch, int64_t consumerId,
int32_t type, int64_t sver, int64_t ever) {
int32_t len = 0;
int32_t code = 0;
@ -394,7 +394,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) {
tEncodeSize(tEncodeMqDataRsp, pRsp, len, code);
} else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code);
tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code);
}
if (code < 0) {
@ -418,7 +418,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) {
tEncodeMqDataRsp(&encoder, pRsp);
} else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) {
tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp);
tEncodeSTaosxRsp(&encoder, pRsp);
}
tEncoderClear(&encoder);

16
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -86,6 +86,22 @@ int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode)
return TSDB_CODE_SUCCESS;
}
void tqSetRestoreVersionInfo(SStreamTask* pTask) {
SCheckpointInfo* pChkInfo = &pTask->chkInfo;
// checkpoint ver is the kept version, handled data should be the next version.
if (pChkInfo->checkpointId != 0) {
pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1;
pChkInfo->processedVer = pChkInfo->checkpointVer;
pTask->execInfo.startCheckpointId = pChkInfo->checkpointId;
tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr,
pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
}
pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer;
}
int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) {
int32_t vgId = pMeta->vgId;
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);

2
source/libs/audit/src/auditMain.c
View File

@ -27,7 +27,7 @@
#include "osMemory.h"
SAudit tsAudit = {0};
char* tsAuditUri = "/audit";
char* tsAuditUri = "/audit_v2";
char* tsAuditBatchUri = "/audit-batch";
int32_t auditInit(const SAuditCfg *pCfg) {

13
source/libs/executor/src/cachescanoperator.c
View File

@ -108,6 +108,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
goto _error;
}
// todd: the pk information should comes from the physical plan
for(int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
SColMatchItem* pItem = taosArrayGet(pInfo->matchInfo.pList, i);
if (pItem->isPk) {
@ -223,8 +224,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
blockDataCleanup(pInfo->pBufferedRes);
taosArrayClear(pInfo->pUidList);
int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferedRes, pInfo->pSlotIds,
pInfo->pDstSlotIds, pInfo->pUidList);
int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferedRes,
pInfo->pSlotIds, pInfo->pDstSlotIds, pInfo->pUidList);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
@ -293,9 +294,11 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
}
if (NULL == pInfo->pLastrowReader) {
code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader,
pTaskInfo->id.str, pInfo->pFuncTypeList, &pInfo->pkCol, pInfo->numOfPks);
code = pInfo->readHandle.api.cacheFn.openReader(
pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, taosArrayGetSize(pInfo->matchInfo.pList),
pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader, pTaskInfo->id.str, pInfo->pFuncTypeList,
&pInfo->pkCol, pInfo->numOfPks);
if (code != TSDB_CODE_SUCCESS) {
pInfo->currentGroupIndex += 1;
taosArrayClear(pInfo->pUidList);

15
source/libs/parser/src/parInsertUtil.c
View File

@ -641,7 +641,7 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) {
}
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* tFields,
int numFields, bool needChangeLength) {
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen) {
void* tmp =
taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid));
STableDataCxt* pTableCxt = NULL;
@ -662,8 +662,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
}
char* p = (char*)data;
// | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
// column length |
// | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each column length |
int32_t version = *(int32_t*)data;
p += sizeof(int32_t);
p += sizeof(int32_t);
@ -689,12 +688,12 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo;
if (tFields != NULL && numFields != numOfCols) {
uError("numFields:%d != numOfCols:%d", numFields, numOfCols);
if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d != raw numOfCols:%d", numFields, numOfCols);
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
if (tFields != NULL && numFields > boundInfo->numOfBound) {
uError("numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound);
if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound);
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
@ -703,7 +702,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema* pColSchema = &pSchema[j];
SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j);
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
uError("type or bytes not equal");
if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d",
pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t)));
ret = TSDB_CODE_INVALID_PARA;
goto end;
}
@ -732,7 +732,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema* pColSchema = &pSchema[j];
if (strcmp(pColSchema->name, tFields[i].name) == 0) {
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
uError("type or bytes not equal");
if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d",
pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t)));
ret = TSDB_CODE_INVALID_PARA;
goto end;
}

3
source/libs/planner/src/planPhysiCreater.c
View File

@ -570,13 +570,14 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu
if (pScanLogicNode->pVgroupList) {
vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
}
int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
if (TSDB_CODE_SUCCESS == code && pScanLogicNode->pFuncTypes != NULL) {
pScan->pFuncTypes = taosArrayInit(taosArrayGetSize(pScanLogicNode->pFuncTypes), sizeof(int32_t));
if (NULL == pScan->pFuncTypes) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pTargetNode = NULL;
int funcTypeIndex = 0;
FOREACH(pTargetNode, ((SScanPhysiNode*)pScan)->pScanCols) {

9
source/libs/stream/src/streamMeta.c
View File

@ -1288,7 +1288,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) {
}
SStreamTask* pTask = *(SStreamTask**)pIter;
stDebug("vgId:%d s-task:%s set closing flag", vgId, pTask->id.idStr);
stDebug("vgId:%d s-task:%s set task closing flag", vgId, pTask->id.idStr);
streamTaskStop(pTask);
}
@ -1649,6 +1649,13 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
return 0;
}
void* p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (p == NULL) { // task does not exists in current vnode, not record the complete info
qError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId);
streamMetaWUnLock(pMeta);
return 0;
}
SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet;
STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready};

71
source/libs/stream/src/streamTask.c
View File

@ -993,16 +993,17 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t
SDownstreamStatusInfo* p = taosArrayGet(pInfo->pList, i);
if (p->taskId == taskId) {
ASSERT(reqId == p->reqId);
p->status = status;
p->rspTs = rspTs;
// count down one, since it is ready now
if (p->status == TASK_DOWNSTREAM_READY) {
if ((p->status != TASK_DOWNSTREAM_READY) && (status == TASK_DOWNSTREAM_READY)) {
*pNotReady = atomic_sub_fetch_32(&pInfo->notReadyTasks, 1);
} else {
*pNotReady = pInfo->notReadyTasks;
}
p->status = status;
p->rspTs = rspTs;
taosThreadMutexUnlock(&pInfo->checkInfoLock);
return TSDB_CODE_SUCCESS;
}
@ -1026,7 +1027,7 @@ int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) {
}
taosThreadMutexUnlock(&pInfo->checkInfoLock);
stDebug("s-task:%s set the in check procedure flag", id);
stDebug("s-task:%s set the in-check-procedure flag", id);
return 0;
}
@ -1038,7 +1039,7 @@ int32_t streamTaskCompleteCheck(STaskCheckInfo* pInfo, const char* id) {
}
int64_t el = taosGetTimestampMs() - pInfo->startTs;
stDebug("s-task:%s check downstream completed, elapsed time:%" PRId64 " ms", id, el);
stDebug("s-task:%s clear the in-check-procedure flag, elapsed time:%" PRId64 " ms", id, el);
pInfo->startTs = 0;
pInfo->inCheckProcess = 0;
@ -1098,27 +1099,36 @@ static void rspMonitorFn(void* param, void* tmrId) {
int64_t now = taosGetTimestampMs();
int64_t el = now - pInfo->startTs;
ETaskStatus state = pStat->state;
int32_t numOfReady = 0;
int32_t numOfFault = 0;
const char* id = pTask->id.idStr;
stDebug("s-task:%s start to do check downstream rsp check", pTask->id.idStr);
stDebug("s-task:%s start to do check downstream rsp check", id);
if (state == TASK_STATUS__STOP || state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) {
if (state == TASK_STATUS__STOP) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d quit from monitor rsp tmr, ref:%d", pTask->id.idStr, pStat->name, vgId, ref);
streamTaskCompleteCheck(pInfo, pTask->id.idStr);
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref);
streamTaskCompleteCheck(pInfo, id);
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false);
return;
}
if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref);
streamTaskCompleteCheck(pInfo, id);
return;
}
taosThreadMutexLock(&pInfo->checkInfoLock);
if (pInfo->notReadyTasks == 0) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", pTask->id.idStr,
pStat->name, vgId, ref);
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat->name,
vgId, ref);
taosThreadMutexUnlock(&pInfo->checkInfoLock);
streamTaskCompleteCheck(pInfo, pTask->id.idStr);
streamTaskCompleteCheck(pInfo, id);
return;
}
@ -1131,13 +1141,12 @@ static void rspMonitorFn(void* param, void* tmrId) {
if (p->status == TASK_DOWNSTREAM_READY) {
numOfReady += 1;
} else if (p->status == TASK_UPSTREAM_NEW_STAGE || p->status == TASK_DOWNSTREAM_NOT_LEADER) {
stDebug("s-task:%s recv status from downstream, task:0x%x, quit from check downstream tasks", pTask->id.idStr,
p->taskId);
stDebug("s-task:%s recv status from downstream, task:0x%x, quit from check downstream tasks", id, p->taskId);
numOfFault += 1;
} else { // TASK_DOWNSTREAM_NOT_READY
if (p->rspTs == 0) { // not response yet
} else { // TASK_DOWNSTREAM_NOT_READY
if (p->rspTs == 0) { // not response yet
ASSERT(p->status == -1);
if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec.
if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec.
taosArrayPush(pTimeoutList, &p->taskId);
} else { // el < CHECK_NOT_RSP_DURATION
// do nothing and continue waiting for their rsps
@ -1148,25 +1157,26 @@ static void rspMonitorFn(void* param, void* tmrId) {
}
}
} else { // unexpected status
stError("s-task:%s unexpected task status:%s during waiting for check rsp", pTask->id.idStr, pStat->name);
stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat->name);
}
int32_t numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList);
int32_t numOfTimeout = (int32_t)taosArrayGetSize(pTimeoutList);
// fault tasks detected, not try anymore
if (((numOfReady + numOfFault + numOfNotReady + numOfTimeout) == taosArrayGetSize(pInfo->pList)) && (numOfFault > 0)) {
if (((numOfReady + numOfFault + numOfNotReady + numOfTimeout) == taosArrayGetSize(pInfo->pList)) &&
(numOfFault > 0)) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug(
"s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart "
"detected, ref:%d",
pTask->id.idStr, pStat->name, vgId, ref);
"detected, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d",
id, pStat->name, vgId, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
taosThreadMutexUnlock(&pInfo->checkInfoLock);
taosArrayDestroy(pNotReadyList);
taosArrayDestroy(pTimeoutList);
streamTaskCompleteCheck(pInfo, pTask->id.idStr);
streamTaskCompleteCheck(pInfo, id);
return;
}
@ -1176,14 +1186,14 @@ static void rspMonitorFn(void* param, void* tmrId) {
stDebug(
"s-task:%s status:%s vgId:%d stopped by other threads to check downstream process, notReady:%d, fault:%d, "
"timeout:%d, ready:%d ref:%d",
pTask->id.idStr, pStat->name, vgId, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
id, pStat->name, vgId, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
taosThreadMutexUnlock(&pInfo->checkInfoLock);
// add the not-ready tasks into the final task status result buf, along with related fill-history task if exists.
streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pInfo->startTs, now, false);
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo;
streamMetaAddTaskLaunchResult(pTask->pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId, pInfo->startTs, now, false);
STaskId* pHId = &pTask->hTaskInfo.id;
streamMetaAddTaskLaunchResult(pTask->pMeta, pHId->streamId, pHId->taskId, pInfo->startTs, now, false);
}
return;
}
@ -1205,7 +1215,7 @@ static void rspMonitorFn(void* param, void* tmrId) {
}
}
stDebug("s-task:%s %d downstream task(s) not ready, send check msg again", pTask->id.idStr, numOfNotReady);
stDebug("s-task:%s %d downstream task(s) not ready, send check msg again", id, numOfNotReady);
}
if (numOfTimeout > 0) {
@ -1225,15 +1235,14 @@ static void rspMonitorFn(void* param, void* tmrId) {
}
}
stDebug("s-task:%s %d downstream tasks timeout, send check msg again, start ts:%" PRId64, pTask->id.idStr,
numOfTimeout, now);
stDebug("s-task:%s %d downstream tasks timeout, send check msg again, start ts:%" PRId64, id, numOfTimeout, now);
}
taosThreadMutexUnlock(&pInfo->checkInfoLock);
taosTmrReset(rspMonitorFn, CHECK_RSP_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr);
stDebug("s-task:%s continue checking rsp in 200ms, notReady:%d, fault:%d, timeout:%d, ready:%d", pTask->id.idStr,
numOfNotReady, numOfFault, numOfTimeout, numOfReady);
stDebug("s-task:%s continue checking rsp in 200ms, notReady:%d, fault:%d, timeout:%d, ready:%d", id, numOfNotReady,
numOfFault, numOfTimeout, numOfReady);
taosArrayDestroy(pNotReadyList);
taosArrayDestroy(pTimeoutList);

4
source/util/src/terror.c
View File

@ -27,8 +27,12 @@ typedef struct {
} STaosError;
static threadlocal int32_t tsErrno;
static threadlocal char tsErrMsgDetail[ERR_MSG_LEN] = {0};
static threadlocal char tsErrMsgReturn[ERR_MSG_LEN] = {0};
int32_t* taosGetErrno() { return &tsErrno; }
char* taosGetErrMsg() { return tsErrMsgDetail; }
char* taosGetErrMsgReturn() { return tsErrMsgReturn; }
#ifdef TAOS_ERROR_C
#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg)},

3
tests/parallel_test/cases.task
View File

@ -320,6 +320,9 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_insert.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_delete.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3

62
tests/pytest/util/csv.py Normal file
View File

@ -0,0 +1,62 @@
import csv
import os
class TDCsv:
def __init__(self):
self.__file_name = ''
self.__file_path = ''
@property
def file_name(self):
return self.__file_name
@file_name.setter
def file_name(self, value):
self.__file_name = value
@property
def file_path(self):
return self.__file_path
@file_path.setter
def file_path(self, value):
self.__file_path = value
@property
def file(self):
if self.file_name and self.file_path:
return os.path.join(self.file_path, self.file_name)
return None
def read(self):
try:
with open(self.file, newline='') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
print(row)
except Exception as errMsg:
raise Exception(errMsg)
def __write_with_quotes(self, csv_writer, data):
for row in data:
row_with_quotes = [f'"{field}"' if isinstance(field, str) else field for field in row]
csv_writer.writerow(row_with_quotes)
def write(self, data: dict):
try:
with open(self.file, 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerows(data)
# self.__write_with_quotes(writer, data)
except Exception as errMsg:
raise Exception(errMsg)
def delete(self):
try:
if os.path.exists(self.file):
os.remove(self.file)
except Exception as errMsg:
raise Exception(errMsg)

224
tests/system-test/1-insert/composite_primary_key_create.py Normal file
View File

@ -0,0 +1,224 @@
from enum import Enum
from util.log import *
from util.sql import *
from util.cases import *
class IllegalDataType(Enum):
NULL = 'null'
BOOL = 'bool'
TINYINT = 'tinyint'
SMALLINT = 'smallint'
FLOAT = 'float'
DOUBLE = 'double'
TIMESTAMP = 'timestamp'
NCHAR = 'nchar(100)'
UTINYINT = 'tinyint unsigned'
USMALLINT = 'smallint unsigned'
JSON = 'json'
VARBINARY = 'varbinary(100)'
DECIMAL = 'decimal'
BLOB = 'blob'
MEDIUMBLOB = 'mediumblob'
GEOMETRY = 'geometry(512)'
EMPTY = '\'\''
SPACE = '\' \''
class LegalDataType(Enum):
INT = 'INT'
UINT = 'INT UNSIGNED'
BIGINT = 'BIGINT'
UBIGINT = 'BIGINT UNSIGNED'
VARCHAR = 'VARCHAR(100)'
BINARY = 'BINARY(100)'
class LegalSpell(Enum):
CASE1 = 'primary key'
CASE2 = 'PRIMARY KEY'
CASE3 = 'Primary Key'
CASE4 = 'PriMary keY'
class IllegalSpell(Enum):
CASE1 = 'primary'
CASE2 = 'key'
CASE3 = 'primay key'
CASE4 = 'primary ky'
CASE5 = 'primarykey'
CASE6 = 'key primary'
CASE7 = 'primay key primay key'
class TableType(Enum):
SUPERTABLE = 0
CHILDTABLE = 1
NORNALTABLE = 2
SHOW_LOG = True
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
self.database = "db_create_composite_primary_key"
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
def prepare_db(self):
tdSql.execute(f"drop database if exists {self.database}")
tdSql.execute(f"create database {self.database}")
tdSql.execute(f"use {self.database}")
def check_pk_definition(self, table_name: str, d_type: LegalDataType, t_type: TableType):
tdSql.query(f"describe {table_name}", show=SHOW_LOG)
tdSql.checkData(1, 3, "PRIMARY KEY")
if d_type == LegalDataType.BINARY:
d_type = LegalDataType.VARCHAR
if t_type == TableType.SUPERTABLE:
expected_value = f"CREATE STABLE `{table_name}` (`ts` TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `pk` {d_type.value} ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium' PRIMARY KEY, `c2` INT ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium') TAGS (`engine` INT)"
elif t_type == TableType.NORNALTABLE:
expected_value = f"CREATE TABLE `{table_name}` (`ts` TIMESTAMP ENCODE 'delta-i' COMPRESS 'lz4' LEVEL 'medium', `pk` {d_type.value} ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium' PRIMARY KEY, `c2` INT ENCODE 'simple8b' COMPRESS 'lz4' LEVEL 'medium')"
tdSql.query(f"show create table {table_name}", show=SHOW_LOG)
result = tdSql.queryResult
tdSql.checkEqual("PRIMARY KEY" in result[0][1], True)
def test_pk_datatype_legal(self, stable_name: str, ctable_name: str, ntable_name: str, dtype: LegalDataType):
# create super table and child table
tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG)
tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG)
tdSql.execute(f"create table {stable_name} (ts timestamp, pk {dtype.value} primary key, c2 int) tags (engine int)", show=SHOW_LOG)
self.check_pk_definition(stable_name, dtype, TableType.SUPERTABLE)
tdSql.execute(f"create table {ctable_name} using {stable_name} tags (0)", show=SHOW_LOG)
tdSql.execute(f"create table {ctable_name}_1 using {stable_name} tags (1) {ctable_name}_2 using {stable_name} tags (2) {ctable_name}_3 using {stable_name} tags (3)", show=SHOW_LOG)
# create normal table
tdSql.execute(f"create table {ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 int)", show=SHOW_LOG)
self.check_pk_definition(ntable_name, dtype, TableType.NORNALTABLE)
def test_pk_datatype_illegal(self, stable_name: str, ntable_name: str, dtype: LegalDataType):
# create super table and child table
tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG)
tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG)
tdSql.error(f"create table {stable_name} (ts timestamp, pk {dtype.value} primary key, c2 int) tags (engine int)", show=SHOW_LOG)
# create normal table
tdSql.error(f"create table {ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 int)", show=SHOW_LOG)
def test_pk_spell_legal(self, stable_name: str, ntable_name: str, pk_spell: LegalSpell):
# create super table and child table
tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG)
tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG)
tdSql.execute(f"create table {stable_name} (ts timestamp, pk int {pk_spell.value}, c2 int) tags (engine int)", show=SHOW_LOG)
# create normal table
tdSql.execute(f"create table {ntable_name} (ts timestamp, pk int {pk_spell.value}, c2 int)", show=SHOW_LOG)
def test_pk_spell_illegal(self, stable_name: str, ntable_name: str, pk_spell: LegalSpell):
# create super table and child table
tdSql.execute(f"drop table if exists {stable_name}", show=SHOW_LOG)
tdSql.execute(f"drop table if exists {ntable_name}", show=SHOW_LOG)
tdSql.error(f"create table {stable_name} (ts timestamp, pk int {pk_spell.value}, c2 int) tags (engine int)", show=SHOW_LOG)
# create normal table
tdSql.error(f"create table {ntable_name} (ts timestamp, pk int {pk_spell.value}, c2 int)", show=SHOW_LOG)
def test_update_pk(self, table_name: str, t_type: TableType):
# create super table and child table
tdSql.execute(f"drop table if exists {table_name}", show=SHOW_LOG)
if t_type == TableType.SUPERTABLE:
tdSql.execute(f"create table {table_name} (ts timestamp, c2 int) tags (engine int)", show=SHOW_LOG)
elif t_type == TableType.NORNALTABLE:
# create normal table
tdSql.execute(f"create table {table_name} (ts timestamp, c2 int)", show=SHOW_LOG)
tdSql.error(f"alter table {table_name} add column pk varchar(100) primary key", show=SHOW_LOG)
tdSql.execute(f"drop table if exists {table_name}", show=SHOW_LOG)
if t_type == TableType.SUPERTABLE:
tdSql.execute(f"create table {table_name} (ts timestamp, pk varchar(200) primary key, c2 int) tags (engine int)", show=SHOW_LOG)
elif t_type == TableType.NORNALTABLE:
# create normal table
tdSql.execute(f"create table {table_name} (ts timestamp, pk varchar(200) primary key, c2 int)", show=SHOW_LOG)
tdSql.error(f"alter table {table_name} add column new_pk varchar(20) primary key", show=SHOW_LOG)
for date_type in IllegalDataType.__members__.items():
tdSql.error(f"alter table {table_name} modify column pk {date_type[1].value}", show=SHOW_LOG)
for date_type in LegalDataType.__members__.items():
tdSql.error(f"alter table {table_name} modify column pk {date_type[1].value}", show=SHOW_LOG)
tdSql.error(f"alter table {table_name} modify column pk varchar(300)", show=SHOW_LOG)
tdSql.error(f"alter table {table_name} rename column pk new_pk", show=SHOW_LOG)
tdSql.error(f"alter table {table_name} drop column pk", show=SHOW_LOG)
def run(self):
tdSql.prepare(replica = self.replicaVar)
self.prepare_db()
# 1.check legal data type
for date_type in LegalDataType.__members__.items():
self.test_pk_datatype_legal('s_table', 'c_table', 'n_table', date_type[1])
# 2.check illegal data type
for date_type in IllegalDataType.__members__.items():
self.test_pk_datatype_illegal('s_table', 'n_table', date_type[1])
# 3.check legal spell
for date_type in LegalSpell.__members__.items():
self.test_pk_spell_legal('s_table', 'n_table', date_type[1])
# 4.check illegal spell
for date_type in IllegalSpell.__members__.items():
self.test_pk_spell_illegal('s_table', 'n_table', date_type[1])
# 5.only define ts and pk columns
# create super table and child table
tdSql.execute(f"drop table if exists s_table", show=SHOW_LOG)
tdSql.execute(f"drop table if exists n_table", show=SHOW_LOG)
tdSql.execute(f"create table s_table (ts timestamp, pk int primary key) tags (engine int)", show=SHOW_LOG)
tdSql.execute(f"create table c_table using s_table tags (1)", show=SHOW_LOG)
tdSql.execute(f"insert into c_table values(now, 1)", show=SHOW_LOG)
tdSql.query(f"select * from s_table", show=SHOW_LOG)
tdSql.checkRows(1)
# create normal table
tdSql.execute(f"create table n_table (ts timestamp, pk int primary key)", show=SHOW_LOG)
tdSql.execute(f"insert into n_table values(now, 1)", show=SHOW_LOG)
tdSql.query(f"select * from n_table", show=SHOW_LOG)
tdSql.checkRows(1)
# 6.mutiple pk & pk not defined as sencod column
tdSql.execute(f"drop table if exists s_table", show=SHOW_LOG)
tdSql.execute(f"drop table if exists n_table", show=SHOW_LOG)
# create super table
tdSql.error(f"create table s_table (ts timestamp, c1 int, pk1 int primary key) tags (engine int)", show=SHOW_LOG)
tdSql.error(f"create table s_table (ts timestamp, pk1 int primary key, pk2 int primary key) tags (engine int)", show=SHOW_LOG)
tdSql.error(f"create table s_table (ts timestamp, pk1 int primary key, c2 int, pk2 int primary key) tags (engine int)", show=SHOW_LOG)
# create normal table
tdSql.error(f"create table n_table (ts timestamp, c1 int, pk1 int primary key)", show=SHOW_LOG)
tdSql.error(f"create table n_table (ts timestamp, pk1 int primary key, pk2 int primary key)", show=SHOW_LOG)
tdSql.error(f"create table n_table (ts timestamp, pk1 int primary key, c2 int, pk2 int primary key)", show=SHOW_LOG)
# 7.add\update\delete pk column is not support
self.test_update_pk('s_table', TableType.SUPERTABLE)
self.test_update_pk('n_table', TableType.NORNALTABLE)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

244
tests/system-test/1-insert/composite_primary_key_delete.py Normal file
View File

@ -0,0 +1,244 @@
from enum import Enum
from util.log import *
from util.sql import *
from util.cases import *
from util.csv import *
import os
import taos
import json
from taos import SmlProtocol, SmlPrecision
from taos.error import SchemalessError
class LegalDataType(Enum):
INT = 'INT'
UINT = 'INT UNSIGNED'
BIGINT = 'BIGINT'
UBIGINT = 'BIGINT UNSIGNED'
VARCHAR = 'VARCHAR(100)'
BINARY = 'BINARY(100)'
class TableType(Enum):
SUPERTABLE = 0
CHILDTABLE = 1
NORNALTABLE = 2
SHOW_LOG = True
STAET_TS = '2023-10-01 00:00:00.000'
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
self.database = "db_insert_composite_primary_key"
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
self.testcasePath = os.path.split(__file__)[0]
self.testcasePath = self.testcasePath.replace('\\', '//')
self.testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
self.stable_name = 's_table'
self.ctable_name = 'c_table'
self.ntable_name = 'n_table'
self.ts_list = {}
def prepare_db(self):
tdSql.execute(f"drop database if exists {self.database}")
tdSql.execute(f"create database {self.database} CACHEMODEL 'none'")
tdSql.execute(f"use {self.database}")
def get_latest_ts(self, table_name: str):
tdSql.query(f'select last(ts) from {table_name}')
now_time = tdSql.queryResult[0][0].strftime("%Y-%m-%d %H:%M:%S.%f")
return f"'{now_time}'"
def prepare_data(self, dtype: LegalDataType):
# drop super table and child table
tdSql.execute(f"drop table if exists {self.stable_name}")
tdSql.execute(f"drop table if exists {self.ntable_name}")
# create super table & child table
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c1 smallint, c2 varchar(10)) tags (engine int)", show=SHOW_LOG)
table_name1 = f'{self.ctable_name}_1'
tdSql.execute(f"insert into {table_name1} using {self.stable_name} tags(1) values(now, 1, '7', '6')", show=SHOW_LOG)
child_ts_1 = self.get_latest_ts(table_name1)
tdSql.execute(f"insert into {table_name1} values({child_ts_1}, 2, '8', '6') ({child_ts_1}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name1} (ts, pk, c2) values({child_ts_1}, 4, '8') ({child_ts_1}, 5, '9')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name1} values(now, 1, '7', '6')", show=SHOW_LOG)
child_ts_2 = self.get_latest_ts(table_name1)
tdSql.execute(f"insert into {table_name1} values({child_ts_2}, 2, '8', '6') ({child_ts_2}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name1} (ts, pk, c2) values({child_ts_2}, 4, '8') ({child_ts_2}, 5, '9')", show=SHOW_LOG)
table_name2 = f'{self.ctable_name}_2'
tdSql.execute(f"insert into {table_name2} using {self.stable_name} tags(2) values(now, 1, '7', '6')", show=SHOW_LOG)
child_ts_3 = self.get_latest_ts(table_name2)
tdSql.execute(f"insert into {table_name2} values({child_ts_3}, 2, '8', '6') ({child_ts_3}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name2} (ts, pk, c2) values({child_ts_3}, 4, '8') ({child_ts_3}, 5, '9')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name2} values(now, 1, '7', '6')", show=SHOW_LOG)
child_ts_4 = self.get_latest_ts(table_name2)
tdSql.execute(f"insert into {table_name2} values({child_ts_4}, 2, '8', '6') ({child_ts_4}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name2} (ts, pk, c2) values({child_ts_4}, 4, '8') ({child_ts_4}, 5, '9')", show=SHOW_LOG)
table_name3 = f'{self.ctable_name}_3'
tdSql.execute(f"insert into {table_name3} using {self.stable_name} tags(3) values(now, 1, '7', '6')", show=SHOW_LOG)
child_ts_5 = self.get_latest_ts(table_name3)
tdSql.execute(f"insert into {table_name3} values({child_ts_5}, 2, '8', '6') ({child_ts_5}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name3} (ts, pk, c2) values({child_ts_5}, 4, '8') ({child_ts_5}, 5, '9')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name3} values(now, 1, '7', '6')", show=SHOW_LOG)
child_ts_6 = self.get_latest_ts(table_name3)
tdSql.execute(f"insert into {table_name3} values({child_ts_6}, 2, '8', '6') ({child_ts_6}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name3} (ts, pk, c2) values({child_ts_6}, 4, '8') ({child_ts_6}, 5, '9')", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(30)
# create normal table
tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c1 smallint, c2 varchar(10))", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG)
normal_ts_1 = self.get_latest_ts(self.ntable_name)
tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_1}, 2, '8', '6') ({normal_ts_1}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_1}, 4, '8') ({normal_ts_1}, 5, '9')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG)
normal_ts_2 = self.get_latest_ts(self.ntable_name)
tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_2}, 2, '8', '6') ({normal_ts_2}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_2}, 4, '8') ({normal_ts_2}, 5, '9')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG)
normal_ts_3 = self.get_latest_ts(self.ntable_name)
tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_3}, 2, '8', '6') ({normal_ts_3}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_3}, 4, '8') ({normal_ts_3}, 5, '9')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG)
normal_ts_4 = self.get_latest_ts(self.ntable_name)
tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_4}, 2, '8', '6') ({normal_ts_4}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_4}, 4, '8') ({normal_ts_4}, 5, '9')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG)
normal_ts_5 = self.get_latest_ts(self.ntable_name)
tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_5}, 2, '8', '6') ({normal_ts_5}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_5}, 4, '8') ({normal_ts_5}, 5, '9')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} values(now, 1, '7', '6')", show=SHOW_LOG)
normal_ts_6 = self.get_latest_ts(self.ntable_name)
tdSql.execute(f"insert into {self.ntable_name} values({normal_ts_6}, 2, '8', '6') ({normal_ts_6}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.ntable_name} (ts, pk, c2) values({normal_ts_6}, 4, '8') ({normal_ts_6}, 5, '9')", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(30)
self.ts_list['child_ts_1'] = child_ts_1
self.ts_list['child_ts_2'] = child_ts_2
self.ts_list['child_ts_3'] = child_ts_3
self.ts_list['child_ts_4'] = child_ts_4
self.ts_list['child_ts_5'] = child_ts_5
self.ts_list['child_ts_6'] = child_ts_6
self.ts_list['normal_ts_1'] = normal_ts_1
self.ts_list['normal_ts_2'] = normal_ts_2
self.ts_list['normal_ts_3'] = normal_ts_3
self.ts_list['normal_ts_4'] = normal_ts_4
self.ts_list['normal_ts_5'] = normal_ts_5
self.ts_list['normal_ts_6'] = normal_ts_6
def test_delete_data(self):
# delete with ts
tdSql.execute(f"delete from {self.stable_name} where ts={self.ts_list['child_ts_1']} ", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(25)
tdSql.execute(f"delete from {self.ctable_name}_1 where ts={self.ts_list['child_ts_2']} ", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(20)
tdSql.execute(f"delete from {self.ntable_name} where ts={self.ts_list['normal_ts_1']} ", show=SHOW_LOG)
tdSql.query(f'select * from {self.ntable_name}')
tdSql.checkRows(25)
# delete with ts range
tdSql.execute(f"delete from {self.stable_name} where ts>{self.ts_list['child_ts_2']} and ts<{self.ts_list['child_ts_4']}", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(15)
tdSql.execute(f"delete from {self.ctable_name}_2 where ts>{self.ts_list['child_ts_2']} and ts<{self.ts_list['child_ts_5']}", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(10)
tdSql.execute(f"delete from {self.ntable_name} where ts>{self.ts_list['normal_ts_2']} and ts<{self.ts_list['normal_ts_5']}", show=SHOW_LOG)
tdSql.query(f'select * from {self.ntable_name}')
tdSql.checkRows(15)
tdSql.execute(f"delete from {self.stable_name}", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
table_name4 = f'{self.ctable_name}_4'
tdSql.execute(f"insert into {table_name4} using {self.stable_name} tags(3) values(now, 1, '7', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name4} values(now+1s, 2, '8', '6') (now+2s, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"delete from {table_name4}", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
tdSql.execute(f"delete from {self.ntable_name}", show=SHOW_LOG)
tdSql.query(f'select * from {self.ntable_name}')
tdSql.checkRows(0)
tdSql.execute(f"delete from {self.stable_name}", show=SHOW_LOG)
tdSql.execute(f"delete from {table_name4}", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
tdSql.execute(f"delete from {self.ntable_name}", show=SHOW_LOG)
tdSql.query(f'select * from {self.ntable_name}')
tdSql.checkRows(0)
def test_delete_data_illegal(self, dtype: LegalDataType):
if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY:
pk_condition_value = '\'1\''
else:
pk_condition_value = '1'
# delete with ts & pk
tdSql.error(f"delete from {self.stable_name} where ts={self.ts_list['child_ts_1']} and pk={pk_condition_value}", show=SHOW_LOG)
tdSql.error(f"delete from {self.ctable_name}_2 where ts={self.ts_list['child_ts_1']} and pk={pk_condition_value}", show=SHOW_LOG)
tdSql.error(f"delete from {self.ntable_name} where ts={self.ts_list['normal_ts_1']} and pk={pk_condition_value}", show=SHOW_LOG)
# delete with ts range & pk
tdSql.error(f"delete from {self.stable_name} where ts>{self.ts_list['child_ts_1']} and ts<{self.ts_list['child_ts_2']} and pk={pk_condition_value}", show=SHOW_LOG)
tdSql.error(f"delete from {self.ctable_name}_2 where ts>{self.ts_list['child_ts_1']} and ts<{self.ts_list['child_ts_2']} and pk={pk_condition_value}", show=SHOW_LOG)
tdSql.error(f"delete from {self.ntable_name} where ts>{self.ts_list['normal_ts_1']} and ts<{self.ts_list['normal_ts_2']} and pk={pk_condition_value}", show=SHOW_LOG)
# delete with pk
tdSql.error(f"delete from {self.stable_name} where pk={pk_condition_value}", show=SHOW_LOG)
tdSql.error(f"delete from {self.ctable_name}_2 where pk={pk_condition_value}", show=SHOW_LOG)
tdSql.error(f"delete from {self.ntable_name} where pk={pk_condition_value}", show=SHOW_LOG)
def _compare_table_data(self, result1, result2, row = 0, col = 0):
for i in range(row):
for j in range(col):
if result1[i][j] != result2[i][j]:
tdSql.checkEqual(False, True)
def run(self):
tdSql.prepare(replica = self.replicaVar)
self.prepare_db()
for date_type in LegalDataType.__members__.items():
self.prepare_data(date_type[1])
self.test_delete_data_illegal(date_type[1])
self.test_delete_data()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

958
tests/system-test/1-insert/composite_primary_key_insert.py Normal file
View File

@ -0,0 +1,958 @@
from enum import Enum
from util.log import *
from util.sql import *
from util.cases import *
from util.csv import *
import os
import taos
import json
from taos import SmlProtocol, SmlPrecision
from taos.error import SchemalessError
class IllegalData(Enum):
NULL = 'null'
# EMPTY = '\'\''
NONE = 'none'
# TRUE = 'true'
# FALSE = 'false'
# class IllegalDataVar(Enum):
# NULL = 'null'
# EMPTY = '\'\''
# NONE = 'none'
# TRUE = 'true'
# FALSE = 'false'
class LegalDataType(Enum):
INT = 'INT'
UINT = 'INT UNSIGNED'
BIGINT = 'BIGINT'
UBIGINT = 'BIGINT UNSIGNED'
VARCHAR = 'VARCHAR(100)'
BINARY = 'BINARY(100)'
class TableType(Enum):
SUPERTABLE = 0
CHILDTABLE = 1
NORNALTABLE = 2
class HasPK(Enum):
NO = 0
YES = 1
SHOW_LOG = True
STAET_TS = '2023-10-01 00:00:00.000'
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
self.database = "db_insert_composite_primary_key"
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
self.testcasePath = os.path.split(__file__)[0]
self.testcasePath = self.testcasePath.replace('\\', '//')
self.testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
# tdSql.execute(f"insert into db4096.ctb00 file '{self.testcasePath}//tableColumn4096csvLength64k.csv'")
self.tdCsv = TDCsv()
self.tdCsv.file_path = self.testcasePath
self.tdCsv.file_name = 'file1.csv'
self.stable_name = 's_table'
self.ctable_name = 'c_table'
self.ntable_name = 'n_table'
def prepare_db(self):
tdSql.execute(f"drop database if exists {self.database}")
tdSql.execute(f"create database {self.database}")
tdSql.execute(f"use {self.database}")
def get_latest_ts(self, table_name: str):
tdSql.query(f'select last(ts) from {table_name}')
now_time = tdSql.queryResult[0][0].strftime("%Y-%m-%d %H:%M:%S.%f")
return f"'{now_time}'"
def test_insert_data(self, dtype: LegalDataType, hasPk: HasPK):
# drop super table and child table
tdSql.execute(f"drop table if exists {self.stable_name}")
tdSql.execute(f"drop table if exists {self.ntable_name}_1")
tdSql.execute(f"drop table if exists {self.ntable_name}_2")
if hasPk:
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value}) tags (engine int)", show=SHOW_LOG)
else:
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value}, c3 {dtype.value}) tags (engine int)", show=SHOW_LOG)
# 1.insert into value through supper table
table_name = f'{self.ctable_name}_1'
tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now, 1, '1')", show=SHOW_LOG)
current_ts1 = self.get_latest_ts(self.stable_name)
tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts1}, 2, '2')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts1}, 3, '3')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now + 1s, 1, '4')", show=SHOW_LOG)
current_ts2 = self.get_latest_ts(self.stable_name)
tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, {current_ts2}, 2, '5')", show=SHOW_LOG)
tdSql.execute(f"insert into {self.stable_name} (tbname, engine, ts, pk) values('{table_name}', 1, now + 2s, 2)", show=SHOW_LOG)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(6)
tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts1}", show=SHOW_LOG)
tdSql.checkRows(3)
tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts2}", show=SHOW_LOG)
tdSql.checkRows(2)
tdSql.query(f"select * from {self.stable_name} where engine = 1 and ts ={current_ts2} and pk = 2", show=SHOW_LOG)
tdSql.checkRows(1)
self._check_select(self.stable_name)
# 2.insert into value through child table
table_name = f'{self.ctable_name}_2'
tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) values(now, 1, '7', '6')", show=SHOW_LOG)
current_ts3 = self.get_latest_ts(table_name)
tdSql.execute(f"insert into {table_name} values({current_ts3}, 2, '8', '6') ({current_ts3}, 3, '9', '6')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) values(now + 2s, 1, '10', '6')", show=SHOW_LOG)
current_ts4 = self.get_latest_ts(table_name)
tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) (ts, pk, c2) values({current_ts4}, 2, '11')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(2) (ts, pk) values(now + 4s, 2)", show=SHOW_LOG)
tdSql.query(f'select * from {table_name}')
tdSql.checkRows(6)
tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts3}", show=SHOW_LOG)
tdSql.checkRows(3)
tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts4}", show=SHOW_LOG)
tdSql.checkRows(2)
tdSql.query(f"select * from {table_name} where engine = 2 and ts ={current_ts4} and pk = 2", show=SHOW_LOG)
tdSql.checkRows(1)
self._check_select(table_name)
# 3.insert value into child table from csv file
data = [
['ts','pk','c2'],
['\'2024-03-29 16:55:42.572\'','1','1','100'],
['\'2024-03-29 16:55:42.572\'','2','2','100'],
['\'2024-03-29 16:55:42.572\'','3','3','100'],
['\'2024-03-29 16:55:43.586\'','1','4','100'],
['\'2024-03-29 16:55:43.586\'','2','5','100'],
['\'2024-03-29 16:55:44.595\'','2','6','100']
]
self.tdCsv.delete()
self.tdCsv.write(data)
table_name = f'{self.ctable_name}_3'
tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from {table_name}')
tdSql.checkRows(6)
tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:42.572'", show=SHOW_LOG)
tdSql.checkRows(3)
tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:43.586'", show=SHOW_LOG)
tdSql.checkRows(2)
tdSql.query(f"select * from {table_name} where engine=3 and ts='2024-03-29 16:55:43.586' and pk=2", show=SHOW_LOG)
tdSql.checkRows(1)
self._check_select(table_name)
# 4.insert value into child table from csv file, create table automatically
data = [
['ts','pk','c2'],
['\'2024-03-28 16:55:42.572\'','1','1','100'],
['\'2024-03-28 16:55:42.572\'','2','2','100'],
['\'2024-03-28 16:55:42.572\'','3','3','100'],
['\'2024-03-28 16:55:43.586\'','1','4','100'],
['\'2024-03-28 16:55:43.586\'','2','5','100'],
['\'2024-03-28 16:55:44.595\'','2','6','100']
]
self.tdCsv.delete()
self.tdCsv.write(data)
table_name = f'{self.ctable_name}_4'
self._check_select(self.stable_name)
tdSql.execute(f"insert into {table_name} using {self.stable_name} tags(4) file '{self.tdCsv.file}'", show=SHOW_LOG)
tdSql.query(f'select * from {table_name}')
tdSql.checkRows(6)
tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:42.572'", show=SHOW_LOG)
tdSql.checkRows(3)
tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:43.586'", show=SHOW_LOG)
tdSql.checkRows(2)
tdSql.query(f"select * from {self.stable_name} where engine=4 and ts='2024-03-28 16:55:43.586' and pk=2", show=SHOW_LOG)
tdSql.checkRows(1)
self._check_select(self.stable_name)
# 5.insert value into normal table from csv file
table_name = f'{self.ntable_name}_1'
if hasPk:
tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG)
else:
tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG)
tdSql.query(f'select * from {table_name}')
tdSql.checkRows(6)
tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:42.572'", show=SHOW_LOG)
tdSql.checkRows(3)
tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:43.586'", show=SHOW_LOG)
tdSql.checkRows(2)
tdSql.query(f"select * from {table_name} where ts='2024-03-28 16:55:43.586' and pk=2", show=SHOW_LOG)
tdSql.checkRows(1)
self._check_select(table_name)
# 6.insert value into normal table
table_name = f'{self.ntable_name}_2'
if hasPk:
tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG)
else:
tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name} values(now, 1, '1', '234')", show=SHOW_LOG)
current_ts1 = self.get_latest_ts(table_name)
tdSql.execute(f"insert into {table_name} values({current_ts1}, 2, '2', '234') ({current_ts1}, 3, '3', '234')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name} values(now + 1s, 1, '4', '234')", show=SHOW_LOG)
current_ts2 = self.get_latest_ts(table_name)
tdSql.execute(f"insert into {table_name} (ts, pk, c2) values({current_ts2}, 2, '5')", show=SHOW_LOG)
tdSql.execute(f"insert into {table_name} (ts, pk) values(now + 2s, 2)", show=SHOW_LOG)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from {table_name}')
tdSql.checkRows(6)
tdSql.query(f"select * from {table_name} where ts ={current_ts1}", show=SHOW_LOG)
tdSql.checkRows(3)
tdSql.query(f"select * from {table_name} where ts ={current_ts2}", show=SHOW_LOG)
tdSql.checkRows(2)
tdSql.query(f"select * from {table_name} where ts ={current_ts2} and pk = 2", show=SHOW_LOG)
tdSql.checkRows(1)
self._check_select(table_name)
def test_insert_data_illegal(self, dtype: LegalDataType, illegal_data: IllegalData):
# drop tables
tdSql.execute(f"drop table if exists {self.stable_name}")
tdSql.execute(f"drop table if exists {self.ntable_name}_1")
tdSql.execute(f"drop table if exists {self.ntable_name}_2")
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG)
# 1.insert into value through supper table
table_name = f'{self.ctable_name}_1'
tdSql.error(f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) values('{table_name}', 1, now, {illegal_data.value}, '1')", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
# 2.insert into value through child table
table_name = f'{self.ctable_name}_2'
tdSql.error(f"insert into {table_name} using {self.stable_name} tags(2) values(now, {illegal_data.value}, '7')", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
# 4.insert value into child table from csv file
data = [
['ts','pk','c2'],
['\'2024-03-29 16:55:42.572\'','1','1'],
['\'2024-03-29 16:55:42.572\'',f'{illegal_data.value}','2'],
['\'2024-03-29 16:55:42.572\'','3','3'],
['\'2024-03-29 16:55:43.586\'','1','4'],
['\'2024-03-29 16:55:43.586\'','2','5'],
['\'2024-03-29 16:55:44.595\'','2','6']
]
self.tdCsv.delete()
self.tdCsv.write(data)
table_name = f'{self.ctable_name}_3'
tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG)
tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG)
tdSql.query(f'select * from {table_name}')
tdSql.checkRows(0)
self._check_select(table_name)
# 5.insert value into child table from csv file, create table automatically
table_name = f'{self.ctable_name}_4'
tdSql.error(f"insert into {table_name} using {self.stable_name} tags(4) file '{self.tdCsv.file}'", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
# 6.insert value into normal table from csv file
table_name = f'{self.ntable_name}_1'
tdSql.execute(f"create table {table_name} using {self.stable_name} tags(3)", show=SHOW_LOG)
tdSql.error(f"insert into {table_name} file '{self.tdCsv.file}'", show=SHOW_LOG)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from {table_name}')
tdSql.checkRows(0)
self._check_select(table_name)
# 7.insert value into normal table
table_name = f'{self.ntable_name}_2'
tdSql.execute(f"create table {table_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG)
tdSql.error(f"insert into {table_name} values(now, {illegal_data.value}, '1')", show=SHOW_LOG)
tdSql.query(f'select * from {table_name}')
tdSql.checkRows(0)
self._check_select(table_name)
def test_insert_select(self, dtype: LegalDataType):
# # 1.pk table to non-pk table, throw error
tdSql.execute(f"drop table if exists source_{self.stable_name}")
tdSql.execute(f"drop table if exists source_{self.ctable_name}")
tdSql.execute(f"drop table if exists source_{self.ntable_name}")
tdSql.execute(f"drop table if exists dest_{self.stable_name}")
tdSql.execute(f"drop table if exists dest_{self.ntable_name}")
tdSql.execute(f"create table source_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG)
tdSql.execute(f"create table source_{self.ctable_name} using source_{self.stable_name} tags(3)", show=SHOW_LOG)
tdSql.execute(f"create table source_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG)
tdSql.execute(f"create table dest_{self.ntable_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG)
tdSql.error(f"insert into dest_{self.ntable_name} select * from source_{self.stable_name})", show=SHOW_LOG)
tdSql.error(f"insert into dest_{self.ntable_name} select * from source_{self.ctable_name})", show=SHOW_LOG)
tdSql.error(f"insert into dest_{self.ntable_name} select * from source_{self.ntable_name})", show=SHOW_LOG)
# 2.non-pk table to pk table
tdSql.execute(f"drop table if exists source_{self.stable_name}")
tdSql.execute(f"drop table if exists source_{self.ctable_name}")
tdSql.execute(f"drop table if exists source_{self.ntable_name}")
tdSql.execute(f"drop table if exists dest_{self.stable_name}")
tdSql.execute(f"drop table if exists dest_{self.ntable_name}")
# create dest super & child table
tdSql.execute(f"create table dest_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG)
tdSql.execute(f"create table dest_{self.ctable_name} using dest_{self.stable_name} tags(3)", show=SHOW_LOG)
# tdSql.execute(f"insert into source_{self.ctable_name} values(now, 1, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG)
# create normal dest table
tdSql.execute(f"create table dest_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG)
# create source table & insert data
tdSql.execute(f"create table source_{self.ntable_name} (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG)
tdSql.execute(f"insert into source_{self.ntable_name} values(now, 1, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG)
tdSql.query(f'select * from source_{self.ntable_name}')
source_data = tdSql.queryResult
tdSql.execute(f"insert into dest_{self.ctable_name} select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from dest_{self.ctable_name}')
dest_data = tdSql.queryResult
self._compare_table_data(source_data, dest_data, 3, 3)
self._check_select(f'dest_{self.ctable_name}')
tdSql.execute(f"delete from dest_{self.ctable_name}", show=SHOW_LOG)
self._check_select(f'dest_{self.ctable_name}')
tdSql.execute(f"insert into dest_{self.ntable_name} select * from source_{self.ntable_name}", show=SHOW_LOG)
tdSql.query(f'select * from dest_{self.ntable_name}')
dest_data = tdSql.queryResult
self._compare_table_data(source_data, dest_data, 3, 3)
self._check_select(f'dest_{self.ntable_name}')
tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG)
self._check_select(f'dest_{self.ntable_name}')
# TD-29363
tdSql.execute(f"drop table if exists source_null")
tdSql.execute(f"create table source_null (ts timestamp, pk {dtype.value}, c2 {dtype.value})", show=SHOW_LOG)
tdSql.execute(f"insert into source_null values(now, null, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG)
tdSql.error(f"insert into dest_{self.ntable_name} values(now, null, 1) (now+1s, 2, 2) (now+2s, 3, 3)", show=SHOW_LOG)
tdSql.error(f"insert into dest_{self.ctable_name} select ts, pk, c2 from source_null", show=SHOW_LOG)
tdSql.error(f"insert into dest_{self.ntable_name} select * from source_null", show=SHOW_LOG)
# 3.pk table to pk table
tdSql.execute(f"drop table if exists source_{self.stable_name}")
tdSql.execute(f"drop table if exists source_{self.ctable_name}")
tdSql.execute(f"drop table if exists source_{self.ntable_name}")
tdSql.execute(f"drop table if exists dest_{self.stable_name}")
tdSql.execute(f"drop table if exists dest_{self.ntable_name}")
# create dest super & child table
tdSql.execute(f"create table dest_{self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value}) tags (engine int)", show=SHOW_LOG)
tdSql.execute(f"create table dest_{self.ctable_name} using dest_{self.stable_name} tags(3)", show=SHOW_LOG)
tdSql.execute(f"insert into dest_{self.ctable_name} values('2024-04-01 17:38:08.764', 1, 1, 100) ('2024-04-01 17:38:08.764', 2, 2, 200) ('2024-04-01 17:38:08.764', 3, 3, 300)", show=SHOW_LOG)
tdSql.execute(f"insert into dest_{self.ctable_name} values('2024-04-02 17:38:08.764', 1, 4, 400) ('2024-04-02 17:38:08.764', 2, 1, 500) ('2024-04-02 17:38:08.764', 3, 1, 600)", show=SHOW_LOG)
# create normal dest table
tdSql.execute(f"create table dest_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG)
tdSql.execute(f"insert into dest_{self.ntable_name} values('2024-04-01 17:38:08.764', 1, 1, 100) ('2024-04-01 17:38:08.764', 2, 2, 200) ('2024-04-01 17:38:08.764', 3, 3, 300)", show=SHOW_LOG)
tdSql.execute(f"insert into dest_{self.ntable_name} values('2024-04-02 17:38:08.764', 1, 4, 400) ('2024-04-02 17:38:08.764', 2, 1, 500) ('2024-04-02 17:38:08.764', 3, 1, 600)", show=SHOW_LOG)
# create source table & insert data
tdSql.execute(f"create table source_{self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 {dtype.value})", show=SHOW_LOG)
tdSql.execute(f"insert into source_{self.ntable_name} values('2024-04-02 17:38:08.764', 1, 4, 800) ('2024-04-02 17:38:08.764', 2, 5, 400) ('2024-04-02 17:38:08.764', 3, 6, 600)", show=SHOW_LOG)
tdSql.execute(f"insert into source_{self.ntable_name} values('2024-04-04 17:38:08.764', 1, 7, 365)", show=SHOW_LOG)
# insert into select
tdSql.execute(f"insert into dest_{self.ctable_name} (ts, pk, c2) select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG)
tdSql.query(f'select * from dest_{self.ctable_name}')
tdSql.checkRows(7)
if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY:
tdSql.query(f"select * from dest_{self.ctable_name} where c2='5' or c2='6'", show=SHOW_LOG)
else:
tdSql.query(f'select * from dest_{self.ctable_name} where c2=5 or c2=6', show=SHOW_LOG)
tdSql.checkRows(2)
self._check_select(f'dest_{self.ctable_name}')
tdSql.execute(f"delete from dest_{self.ctable_name}", show=SHOW_LOG)
self._check_select(f'dest_{self.ctable_name}')
tdSql.execute(f"insert into dest_{self.ntable_name} (ts, pk, c2) select ts, pk, c2 from source_{self.ntable_name}", show=SHOW_LOG)
tdSql.query(f'select * from dest_{self.ntable_name}')
tdSql.checkRows(7)
if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY:
tdSql.query(f"select * from dest_{self.ntable_name} where c2='5' or c2='6'")
else:
tdSql.query(f'select * from dest_{self.ntable_name} where c2=5 or c2=6')
tdSql.checkRows(2)
self._check_select(f'dest_{self.ntable_name}')
tdSql.execute(f"delete from dest_{self.ntable_name}", show=SHOW_LOG)
self._check_select(f'dest_{self.ntable_name}')
def test_schemaless_error(self):
# 5.1.insert into values via influxDB
lines = ["meters,location=California.LosAngeles,groupid=2 current=11i32,voltage=221,phase=0.28 1648432611249000",
"meters,location=California.LosAngeles,groupid=2 current=13i32,voltage=223,phase=0.29 1648432611249000",
"meters,location=California.LosAngeles,groupid=3 current=10i32,voltage=223,phase=0.29 1648432611249300",
"meters,location=California.LosAngeles,groupid=3 current=11i32,voltage=221,phase=0.35 1648432611249300",
]
try:
conn = taos.connect()
conn.execute("drop database if exists influxDB")
conn.execute("CREATE DATABASE influxDB precision 'us'")
conn.execute("USE influxDB")
conn.execute("CREATE STABLE `meters` (`_ts` TIMESTAMP, `current` int primary key, `voltage` DOUBLE, `phase` DOUBLE) TAGS (`location` NCHAR(32), `groupid` NCHAR(2))")
conn.execute("create table t_be97833a0e1f523fcdaeb6291d6fdf27 using meters tags('California.LosAngeles', 2)")
conn.execute("create table t_10b65f71ff8970369c8c18de0d6be028 using meters tags('California.LosAngeles', 3)")
conn.schemaless_insert(lines, SmlProtocol.LINE_PROTOCOL, SmlPrecision.MICRO_SECONDS)
tdSql.checkEqual(False, True)
except SchemalessError as errMsg:
tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True)
# 5.2.insert into values via OpenTSDB
lines = ["meters.current 1648432611249 10i32 location=California.SanFrancisco groupid=2",
"meters.current 1648432611250 12i32 location=California.SanFrancisco groupid=2",
"meters.current 1648432611249 10i32 location=California.LosAngeles groupid=3",
"meters.current 1648432611250 11i32 location=California.LosAngeles groupid=3",
"meters.voltage 1648432611249 219i32 location=California.SanFrancisco groupid=2",
"meters.voltage 1648432611250 218i32 location=California.SanFrancisco groupid=2",
"meters.voltage 1648432611249 221i32 location=California.LosAngeles groupid=3",
"meters.voltage 1648432611250 217i32 location=California.LosAngeles groupid=3",
]
try:
conn = taos.connect()
conn.execute("drop database if exists OpenTSDB")
conn.execute("CREATE DATABASE OpenTSDB precision 'us'")
conn.execute("USE OpenTSDB")
conn.execute("CREATE STABLE `meters_current` (`_ts` TIMESTAMP, `_value` INT primary key) TAGS (`location` NCHAR(32), `groupid` NCHAR(2))")
conn.execute("CREATE TABLE `t_c66ea0b2497be26ca9d328b59c39dd61` USING `meters_current` (`location`, `groupid`) TAGS ('California.LosAngeles', '3')")
conn.execute("CREATE TABLE `t_e71c6cf63cfcabb0e261886adea02274` USING `meters_current` (`location`, `groupid`) TAGS ('California.SanFrancisco', '2')")
conn.schemaless_insert(lines, SmlProtocol.TELNET_PROTOCOL, SmlPrecision.NOT_CONFIGURED)
tdSql.checkEqual(False, True)
except SchemalessError as errMsg:
tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True)
# 5.3.insert into values via OpenTSDB Json
lines = [{"metric": "meters.current", "timestamp": 1648432611249, "value": "a32", "tags": {"location": "California.SanFrancisco", "groupid": 2}}]
try:
conn = taos.connect()
conn.execute("drop database if exists OpenTSDBJson")
conn.execute("CREATE DATABASE OpenTSDBJson")
conn.execute("USE OpenTSDBJson")
conn.execute("CREATE STABLE `meters_current` (`_ts` TIMESTAMP, `_value` varchar(10) primary key) TAGS (`location` VARCHAR(32), `groupid` DOUBLE)")
conn.execute("CREATE TABLE `t_71d176bfc4c952b64d30d719004807a0` USING `meters_current` (`location`, `groupid`) TAGS ('California.SanFrancisco', 2.000000e+00)")
# global lines
lines = json.dumps(lines)
# note: the first parameter must be a list with only one element.
conn.schemaless_insert([lines], SmlProtocol.JSON_PROTOCOL, SmlPrecision.NOT_CONFIGURED)
tdSql.checkEqual(False, True)
except SchemalessError as errMsg:
tdSql.checkEqual(errMsg.msg == 'Can not insert data into table with primary key', True)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
def test_insert_values_special(self, dtype: LegalDataType):
# 4.insert into values without ts column
# drop tables
tdSql.execute(f"drop table if exists {self.stable_name}")
tdSql.execute(f"drop table if exists {self.ntable_name}")
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 binary(10)) tags (engine int)", show=SHOW_LOG)
# # 4.1.insert into value through supper table
tdSql.error(f"insert into {self.stable_name} (tbname, engine, pk, c2) values('{self.ctable_name}', 1, '1', '1')", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
# # 4.2.insert into value through child table
tdSql.error(f"insert into {self.ctable_name} using {self.stable_name} tags(2) (pk, c2) values('1', '7')", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
# # 4.3.insert value into normal table
tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG)
tdSql.error(f"insert into {self.ntable_name} (pk, c2) values('1', '1')", show=SHOW_LOG)
tdSql.query(f'select * from {self.ntable_name}')
tdSql.checkRows(0)
self._check_select(self.ntable_name)
# 5.insert into values without pk column
# drop tables
tdSql.execute(f"drop table if exists {self.stable_name}")
tdSql.execute(f"drop table if exists {self.ntable_name}")
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 binary(10)) tags (engine int)", show=SHOW_LOG)
# # 5.1.insert into value through supper table
tdSql.error(f"insert into {self.stable_name} (tbname, engine, ts, c2) values('{self.ctable_name}', 1, now, '1')", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
# # 5.2.insert into value through child table
tdSql.error(f"insert into {self.ctable_name} using {self.stable_name} tags(2) (ts, c2) values(now, '7')", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
# # 5.3.insert value into normal table
tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 varchar(20))", show=SHOW_LOG)
tdSql.error(f"insert into {self.ntable_name} (ts, c2) values(now, '1')", show=SHOW_LOG)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from {self.ntable_name}')
tdSql.checkRows(0)
self._check_select(self.ntable_name)
def test_insert_into_mutiple_tables(self, dtype: LegalDataType):
# drop super table and child table
tdSql.execute(f"drop table if exists {self.stable_name}")
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG)
# 1.insert into value by supper table syntax
error_sql = f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) " \
f"values('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 1, 100) " \
f"('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 2, 200) " \
f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 1, 300) " \
f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 2, 400) " \
f"('{self.ctable_name}_3', 3, '2021-07-15 14:06:34.630', null, 500)"
tdSql.error(error_sql, show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
sql = f"insert into {self.stable_name} (tbname, engine, ts, pk, c2) " \
f"values('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 1, 100) " \
f"('{self.ctable_name}_1', 1, '2021-07-13 14:06:34.630', 2, 200) " \
f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 1, 300) " \
f"('{self.ctable_name}_2', 2, '2021-07-14 14:06:34.630', 2, 400) " \
f"('{self.ctable_name}_3', 3, '2021-07-15 14:06:34.630', 1, 500)"
tdSql.execute(sql, show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(5)
tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630'", show=SHOW_LOG)
tdSql.checkRows(2)
tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630' and pk=2", show=SHOW_LOG)
tdSql.checkRows(1)
self._check_select(self.stable_name)
# 2.insert into value and create table automatically
tdSql.execute(f"drop table if exists {self.stable_name}")
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG)
error_sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \
f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " \
f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) values('2021-07-15 14:06:34.630', null, 500)"
tdSql.error(error_sql, show=SHOW_LOG)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \
f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) " \
f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) values('2021-07-15 14:06:34.630', 1, 500)"
tdSql.execute(sql, show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(5)
tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630'", show=SHOW_LOG)
tdSql.checkRows(2)
tdSql.query(f"select * from {self.stable_name} where ts='2021-07-13 14:06:34.630' and pk=2", show=SHOW_LOG)
tdSql.checkRows(1)
self._check_select(self.stable_name)
# 3.insert value into child table from csv file, create table automatically
tdSql.execute(f"drop table if exists {self.stable_name}")
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG)
error_data = [
['ts','pk','c2'],
['\'2024-03-29 16:55:42.572\'','1','1'],
['\'2024-03-29 16:55:42.572\'','2','2'],
['\'2024-03-29 16:55:42.572\'','null','3'],
['\'2024-03-29 16:55:43.586\'','1','4'],
['\'2024-03-29 16:55:43.586\'','2','5'],
['\'2024-03-29 16:55:44.595\'','2','6']
]
self.tdCsv.delete()
self.tdCsv.write(error_data)
sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) FILE '{self.tdCsv.file}' " \
f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) FILE '{self.tdCsv.file}' " \
f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) FILE '{self.tdCsv.file}'"
tdSql.error(sql, show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
self._check_select(self.stable_name)
data = [
['ts','pk','c2'],
['\'2024-03-29 16:55:42.572\'','1','1'],
['\'2024-03-29 16:55:42.572\'','2','2'],
['\'2024-03-29 16:55:42.572\'','2','3'],
['\'2024-03-29 16:55:43.586\'','1','4'],
['\'2024-03-29 16:55:43.586\'','2','5'],
['\'2024-03-29 16:55:44.595\'','2','6']
]
self.tdCsv.delete()
self.tdCsv.write(data)
sql = f"insert into {self.ctable_name}_1 using {self.stable_name} tags(1) FILE '{self.tdCsv.file}'" \
f"{self.ctable_name}_2 using {self.stable_name} (engine) tags(2) FILE '{self.tdCsv.file}'" \
f"{self.ctable_name}_3 using {self.stable_name} (engine) tags(3) FILE '{self.tdCsv.file}'"
tdSql.execute(sql, show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(15)
tdSql.query(f"select * from {self.stable_name} where ts='2024-03-29 16:55:42.572'", show=SHOW_LOG)
tdSql.checkRows(6)
tdSql.query(f"select * from {self.stable_name} where ts='2024-03-29 16:55:42.572' and pk=2", show=SHOW_LOG)
tdSql.checkRows(3)
self._check_select(self.stable_name)
# 6.insert value into normal table
tdSql.execute(f"drop table if exists {self.ntable_name}_1")
tdSql.execute(f"drop table if exists {self.ntable_name}_2")
tdSql.execute(f"create table {self.ntable_name}_1 (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG)
tdSql.execute(f"create table {self.ntable_name}_2 (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG)
sql = f"insert into {self.ntable_name}_1 values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \
f"{self.ntable_name}_2 values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) "
tdSql.execute(sql, show=SHOW_LOG)
tdSql.query(f'select * from {self.ntable_name}_1')
tdSql.checkRows(2)
tdSql.query(f"select * from {self.ntable_name}_2 where ts='2021-07-14 14:06:34.630' and pk=2", show=SHOW_LOG)
tdSql.checkRows(1)
self._check_select(self.ntable_name)
# 7. insert value into child and normal table
tdSql.execute(f"drop table if exists {self.stable_name}")
tdSql.execute(f"drop table if exists {self.ctable_name}")
tdSql.execute(f"drop table if exists {self.ntable_name}")
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}) tags (engine int)", show=SHOW_LOG)
tdSql.execute(f"create table {self.ctable_name} using {self.stable_name} tags (1)", show=SHOW_LOG)
tdSql.execute(f"create table {self.ntable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG)
error_sql = f"insert into {self.ctable_name} values('2021-07-13 14:06:34.630', null, 100) ('2021-07-13 14:06:34.630', 2, 200) " \
f"{self.ntable_name} values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) "
sql = f"insert into {self.ctable_name} values('2021-07-13 14:06:34.630', 1, 100) ('2021-07-13 14:06:34.630', 2, 200) " \
f"{self.ntable_name} values('2021-07-14 14:06:34.630', 1, 300) ('2021-07-14 14:06:34.630', 2, 400) "
tdSql.error(error_sql, show=SHOW_LOG)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(0)
tdSql.query(f'select * from {self.ntable_name}')
tdSql.checkRows(0)
tdSql.execute(sql, show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(2)
tdSql.query(f'select * from {self.ntable_name}')
tdSql.checkRows(2)
self._check_select(self.stable_name)
self._check_select(self.ntable_name)
def test_stmt(self, dtype: LegalDataType):
tdSql.execute(f"drop table if exists {self.stable_name}", show=SHOW_LOG)
tdSql.execute(f"create table {self.stable_name} (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value}, c3 float) tags (engine int)", show=SHOW_LOG)
sql = f"INSERT INTO ? USING {self.stable_name} TAGS(?) VALUES (?,?,?,?)"
conn = taos.connect()
conn.select_db(self.database)
stmt = conn.statement(sql)
tbname = f"d1001"
tags = taos.new_bind_params(1)
tags[0].int([2])
stmt.set_tbname_tags(tbname, tags)
params = taos.new_bind_params(4)
params[0].timestamp((1626861392589, 1626861392589, 1626861392592))
if dtype == LegalDataType.INT :
params[1].int((10, 12, 12))
params[2].int([194, 200, 201])
elif dtype == LegalDataType.UINT:
params[1].int_unsigned((10, 12, 12))
params[2].int_unsigned([194, 200, 201])
elif dtype == LegalDataType.BIGINT:
params[1].bigint((10, 12, 12))
params[2].bigint([194, 200, 201])
elif dtype == LegalDataType.UBIGINT:
params[1].bigint_unsigned((10, 12, 12))
params[2].bigint_unsigned([194, 200, 201])
elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY:
params[1].binary(("s10", "s12", "s12"))
params[2].binary(["s194", "s200", "s201"])
params[3].float([0.31, 0.33, 0.31])
stmt.bind_param_batch(params)
stmt.execute()
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(3)
self._check_select(self.stable_name)
params = taos.new_bind_params(4)
params[0].timestamp((1626861392589))
if dtype == LegalDataType.INT :
params[1].int((11))
params[2].int([199])
elif dtype == LegalDataType.UINT:
params[1].int_unsigned((11))
params[2].int_unsigned([199])
elif dtype == LegalDataType.BIGINT:
params[1].bigint((11))
params[2].bigint([199])
elif dtype == LegalDataType.UBIGINT:
params[1].bigint_unsigned((11))
params[2].bigint_unsigned([199])
elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY:
params[1].binary(("s11"))
params[2].binary(["s199"])
params[3].float([0.31])
stmt.bind_param(params)
stmt.execute()
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(4)
self._check_select(self.stable_name)
params = taos.new_bind_params(4)
params[0].timestamp((1626861392589))
if dtype == LegalDataType.INT :
params[1].int((11))
params[2].int([1000])
elif dtype == LegalDataType.UINT:
params[1].int_unsigned((11))
params[2].int_unsigned([1000])
elif dtype == LegalDataType.BIGINT:
params[1].bigint((11))
params[2].bigint([1000])
elif dtype == LegalDataType.UBIGINT:
params[1].bigint_unsigned((11))
params[2].bigint_unsigned([1000])
elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY:
params[1].binary(("s11"))
params[2].binary(["1000"])
params[3].float([0.31])
stmt.bind_param(params)
stmt.execute()
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(4)
self._check_select(self.stable_name)
params = taos.new_bind_params(4)
params[0].timestamp((1626861392589))
if dtype == LegalDataType.INT :
params[1].int(None)
params[2].int([199])
elif dtype == LegalDataType.UINT:
params[1].int_unsigned(None)
params[2].int_unsigned([199])
elif dtype == LegalDataType.BIGINT:
params[1].bigint(None)
params[2].bigint([199])
elif dtype == LegalDataType.UBIGINT:
params[1].bigint_unsigned(None)
params[2].bigint_unsigned([199])
elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY:
params[1].binary(None)
params[2].binary(["s199"])
params[3].float([0.31])
try:
stmt.bind_param(params)
tdSql.checkEqual(False, True)
except Exception as errMsg:
pass
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(4)
self._check_select(self.stable_name)
params = taos.new_bind_params(4)
params[0].timestamp((1626861392589, 1626861392589, ))
if dtype == LegalDataType.INT :
params[1].int((None, 18,))
params[2].int([194, 200])
elif dtype == LegalDataType.UINT:
params[1].int_unsigned((None, 18))
params[2].int_unsigned([194, 200])
elif dtype == LegalDataType.BIGINT:
params[1].bigint((None, 18))
params[2].bigint([194, 200])
elif dtype == LegalDataType.UBIGINT:
params[1].bigint_unsigned((None, 18))
params[2].bigint_unsigned([194, 200])
elif dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY:
params[1].binary((None, "s18"))
params[2].binary(["s194", "s200"])
params[3].float([0.31, 0.33, 0.31])
try:
stmt.bind_param(params)
tdSql.checkEqual(False, True)
except Exception as errMsg:
pass
tdSql.query(f'select * from {self.stable_name}')
tdSql.checkRows(4)
self._check_select(self.stable_name)
if dtype == LegalDataType.VARCHAR or dtype == LegalDataType.BINARY:
tdSql.query(f'select * from {self.stable_name} where pk="s11"')
tdSql.checkEqual(tdSql.queryResult[0][2] == '1000', True)
else:
tdSql.query(f'select * from {self.stable_name} where pk=11')
tdSql.checkEqual(tdSql.queryResult[0][2] == 1000, True)
stmt.close()
def test_implicit_conversion(self, dtype: LegalDataType):
tdSql.execute(f"drop table if exists dest_table", show=SHOW_LOG)
tdSql.execute(f"create table dest_table (ts timestamp, pk {dtype.value} primary key, c2 {dtype.value})", show=SHOW_LOG)
for type in LegalDataType:
if type == dtype:
continue
tdSql.execute(f"drop table if exists source_table", show=SHOW_LOG)
tdSql.execute(f"create table source_table (ts timestamp, pk {dtype.value} primary key, c2 int)", show=SHOW_LOG)
tdSql.execute(f"insert into source_table values(now, 100, 1000)", show=SHOW_LOG)
tdSql.execute(f"insert into dest_table select * from source_table", show=SHOW_LOG)
tdSql.execute(f"flush database {self.database}", show=SHOW_LOG)
tdSql.query(f'select * from dest_table where c2=1000')
tdSql.checkRows(1)
tdSql.execute(f"delete from dest_table", show=SHOW_LOG)
self._check_select('dest_table')
def _compare_table_data(self, result1, result2, row = 0, col = 0):
for i in range(row):
for j in range(col):
if result1[i][j] != result2[i][j]:
tdSql.checkEqual(False, True)
def _check_select(self, table_nam: str):
tdSql.query(f'select count(*) from {table_nam} ')
tdSql.query(f'select * from {table_nam}')
tdSql.query(f'select last_row(*) from {table_nam}')
tdSql.query(f'select last_row(*) from {table_nam} group by tbname')
tdSql.query(f'select first(*) from {table_nam}')
tdSql.query(f'select first(*) from {table_nam} group by tbname')
tdSql.query(f'select last(*) from {table_nam}')
tdSql.query(f'select last(*) from {table_nam} group by tbname')
tdSql.query(f'select ts, last(pk) from {table_nam} order by ts')
tdSql.query(f'select * from {table_nam} order by ts asc')
tdSql.query(f'select * from {table_nam} order by ts desc')
tdSql.query(f'select * from {table_nam} order by pk asc')
tdSql.query(f'select * from {table_nam} order by pk desc')
tdSql.query(f'select * from {table_nam} order by ts asc, pk desc')
tdSql.query(f'select * from {table_nam} order by ts desc, pk asc')
def run(self):
tdSql.prepare(replica = self.replicaVar)
self.prepare_db()
for date_type in LegalDataType.__members__.items():
tdLog.info(f'<dateType={date_type}>')
# 1.insert into value with pk - pass
tdLog.info('[1.insert into value with pk]')
self.test_insert_data(date_type[1], HasPK.YES)
# 2.insert into value without pk - pass
tdLog.info('[2.insert into value without pk]')
self.test_insert_data(date_type[1], HasPK.NO)
# 3.insert into illegal data - pass
tdLog.info('[3.insert into illegal data]')
for illegal_data in IllegalData.__members__.items():
self.test_insert_data_illegal(date_type[1], illegal_data[1])
# 4. insert into select - pass
tdLog.info('[4. insert into select]')
self.test_insert_select(date_type[1])
# 5. insert into values special cases - pass
tdLog.info('[5. insert into values special cases]')
self.test_insert_values_special(date_type[1])
# 6. test implicit conversion - pass
tdLog.info('[6. test implicit conversion]')
self.test_implicit_conversion(date_type[1])
# 7. insert into value to mutiple tables - pass
tdLog.info('[7. insert into value to mutiple tables]')
self.test_insert_into_mutiple_tables(date_type[1])
# 8. stmt wait for test!!!!
tdLog.info('[8. stmt wait for test]')
self.test_stmt(date_type[1])
# 9. insert data by schemaless model is not allowed - pass
tdLog.info('[9. insert data by schemaless model is not allowed]')
self.test_schemaless_error()
# while(True):
# self.test_stmt(LegalDataType.VARCHAR)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

5
tests/system-test/7-tmq/tmq_primary_key.py
View File

@ -81,6 +81,8 @@ class TDTestCase:
finally:
consumer.close()
time.sleep(4) # wait for heart beat
tdSql.query(f'show subscriptions;')
sub = tdSql.getData(0, 4);
print(sub)
@ -190,6 +192,7 @@ class TDTestCase:
finally:
consumer.close()
time.sleep(4) # wait for heart beat
tdSql.query(f'show subscriptions;')
sub = tdSql.getData(0, 4);
print(sub)
@ -298,6 +301,7 @@ class TDTestCase:
finally:
consumer.close()
time.sleep(4) # wait for heart beat
tdSql.query(f'show subscriptions;')
sub = tdSql.getData(0, 4);
print(sub)
@ -406,6 +410,7 @@ class TDTestCase:
finally:
consumer.close()
time.sleep(4) # wait for heart beat
tdSql.query(f'show subscriptions;')
sub = tdSql.getData(0, 4);
print(sub)

12
utils/test/c/tmq_taosx_ci.c
View File

@ -1000,6 +1000,17 @@ void testConsumeExcluded(int topic_type){
}
taos_free_result(pRes);
}
void testDetailError(){
tmq_raw_data raw = {0};
raw.raw_type = 2;
int32_t code = tmq_write_raw((TAOS *)1, raw);
ASSERT(code);
const char *err = tmq_err2str(code);
char* tmp = strstr(err, "Invalid parameters,detail:taos:0x1 or data");
ASSERT(tmp != NULL);
}
int main(int argc, char* argv[]) {
for (int32_t i = 1; i < argc; i++) {
if (strcmp(argv[i], "-c") == 0) {
@ -1036,5 +1047,6 @@ int main(int argc, char* argv[]) {
testConsumeExcluded(1);
testConsumeExcluded(2);
testDetailError();
taosCloseFile(&g_fp);
}