Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/TD-29367-8

This commit is contained in:
Hongze Cheng 2024-07-20 12:26:41 +08:00
commit f0b2557ec1
17 changed files with 439 additions and 324 deletions

View File

@ -401,7 +401,7 @@ pipeline {
}
}
stage('linux test') {
agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || worker03 || slave215 || slave217 || slave219 "}
agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "}
options { skipDefaultCheckout() }
when {
changeRequest()
@ -454,7 +454,7 @@ pipeline {
cd ${WKC}/tests/parallel_test
export DEFAULT_RETRY_TIME=2
date
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 900 ''' + extra_param + '''
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 1200 ''' + extra_param + '''
'''
}
}

View File

@ -192,6 +192,7 @@ typedef struct SBoundColInfo {
int16_t* pColIndex; // bound index => schema index
int32_t numOfCols;
int32_t numOfBound;
bool hasBoundCols;
} SBoundColInfo;
typedef struct STableColsData {

View File

@ -556,6 +556,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C)
#define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D)
#define TSDB_CODE_TDB_INCONSISTENT_DB_ID TAOS_DEF_ERROR_CODE(0, 0x061E)
// query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)

View File

@ -190,10 +190,10 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) {
task->prev->next = task->next;
task->next->prev = task->prev;
if (task->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel,
.arg = task->arg,
});
TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel,
.arg = task->arg,
}));
}
vnodeAsyncTaskDone(async, task);
}
@ -206,6 +206,9 @@ static void *vnodeAsyncLoop(void *arg) {
SVWorker *worker = (SVWorker *)arg;
SVAsync *async = worker->async;
SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo));
if (cancelArray == NULL) {
return NULL;
}
setThreadName(async->label);
@ -466,7 +469,7 @@ int32_t vnodeAsyncOpen(int32_t numOfThreads) {
vnodeAsyncSetWorkers(2, numOfThreads);
_exit:
return 0;
return code;
}
int32_t vnodeAsyncClose() {
@ -748,10 +751,10 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
task->prev->next = task->next;
task->next->prev = task->prev;
if (task->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel,
.arg = task->arg,
});
TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel,
.arg = task->arg,
}));
}
vnodeAsyncTaskDone(async, task);
}
@ -763,10 +766,10 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
channel->scheduled->prev->next = channel->scheduled->next;
channel->scheduled->next->prev = channel->scheduled->prev;
if (channel->scheduled->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = channel->scheduled->cancel,
.arg = channel->scheduled->arg,
});
TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = channel->scheduled->cancel,
.arg = channel->scheduled->arg,
}));
}
vnodeAsyncTaskDone(async, channel->scheduled);
}

View File

@ -16,13 +16,12 @@
#include "vnd.h"
/* ------------------------ STRUCTURES ------------------------ */
static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) {
static int32_t vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) {
SVBufPool *pPool;
pPool = taosMemoryMalloc(sizeof(SVBufPool) + size);
if (pPool == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
memset(pPool, 0, sizeof(SVBufPool));
@ -44,14 +43,12 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPoo
pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock));
if (!pPool->lock) {
taosMemoryFree(pPool);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
if (taosThreadSpinInit(pPool->lock, 0) != 0) {
taosMemoryFree((void *)pPool->lock);
taosMemoryFree(pPool);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
} else {
pPool->lock = NULL;
@ -77,10 +74,11 @@ int vnodeOpenBufPool(SVnode *pVnode) {
for (int i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) {
// create pool
if (vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i])) {
int32_t code;
if ((code = vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i]))) {
vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno));
vnodeCloseBufPool(pVnode);
return -1;
return code;
}
// add to free list
@ -274,8 +272,6 @@ _exit:
}
int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) {
int32_t code = 0;
taosThreadMutexLock(&pPool->mutex);
pQNode->pNext = pPool->qList.pNext;
@ -285,9 +281,7 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) {
pPool->nQuery++;
taosThreadMutexUnlock(&pPool->mutex);
_exit:
return code;
return 0;
}
void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) {

View File

@ -394,8 +394,7 @@ int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) {
}
if (hashValue < pVnode->config.hashBegin || hashValue > pVnode->config.hashEnd) {
terrno = TSDB_CODE_VND_HASH_MISMATCH;
return -1;
return terrno = TSDB_CODE_VND_HASH_MISMATCH;
}
return 0;

View File

@ -129,17 +129,17 @@ void initTqAPI(SStoreTqReader* pTq) {
pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable;
pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed;
pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it
// pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it
pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it
// pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it
pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it
pTq->tqGetResultBlock = tqGetResultBlock;
// pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
// pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
pTq->tqGetResultBlockTime = tqGetResultBlockTime;
pTq->tqGetStreamExecProgress = tqGetStreamExecInfo;
}
}
void initStateStoreAPI(SStateStore* pStore) {
pStore->streamFileStateInit = streamFileStateInit;

View File

@ -19,31 +19,19 @@
static volatile int32_t VINIT = 0;
int vnodeInit(int nthreads) {
int32_t init;
init = atomic_val_compare_exchange_32(&VINIT, 0, 1);
if (init) {
if (atomic_val_compare_exchange_32(&VINIT, 0, 1)) {
return 0;
}
if (vnodeAsyncOpen(nthreads) != 0) {
return -1;
}
if (walInit() < 0) {
return -1;
}
TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads));
TAOS_CHECK_RETURN(walInit());
return 0;
}
void vnodeCleanup() {
int32_t init = atomic_val_compare_exchange_32(&VINIT, 1, 0);
if (init == 0) return;
// set stop
if (atomic_val_compare_exchange_32(&VINIT, 1, 0) == 0) return;
vnodeAsyncClose();
walCleanUp();
smaCleanUp();
}

File diff suppressed because it is too large Load Diff

View File

@ -45,6 +45,7 @@ int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema
void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname,
SArray *tagName, uint8_t tagNum, int32_t ttl);
int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo);
void insResetBoundColsInfo(SBoundColInfo *pInfo);
void insInitColValues(STableMeta *pTableMeta, SArray *aColValues);
void insCheckTableDataOrder(STableDataCxt *pTableCxt, SRowKey *rowKey);
int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta,

View File

@ -183,6 +183,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E
}
pBoundInfo->numOfBound = 0;
pBoundInfo->hasBoundCols = true;
bool hasPK = pTableMeta->tableInfo.numOfPKs;
int16_t numOfBoundPKs = 0;
@ -1379,6 +1380,8 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp
if (NULL != pStmt->pBoundCols) {
return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo);
} else if (pTableCxt->boundColsInfo.hasBoundCols) {
insResetBoundColsInfo(&pTableCxt->boundColsInfo);
}
return TSDB_CODE_SUCCESS;

View File

@ -180,6 +180,7 @@ void insInitColValues(STableMeta* pTableMeta, SArray* aColValues) { initColValue
int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) {
pInfo->numOfCols = numOfBound;
pInfo->numOfBound = numOfBound;
pInfo->hasBoundCols = false;
pInfo->pColIndex = taosMemoryCalloc(numOfBound, sizeof(int16_t));
if (NULL == pInfo->pColIndex) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -190,6 +191,14 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) {
return TSDB_CODE_SUCCESS;
}
void insResetBoundColsInfo(SBoundColInfo* pInfo) {
pInfo->numOfBound = pInfo->numOfCols;
pInfo->hasBoundCols = false;
for (int32_t i = 0; i < pInfo->numOfCols; ++i) {
pInfo->pColIndex[i] = i;
}
}
void insCheckTableDataOrder(STableDataCxt* pTableCxt, SRowKey* rowKey) {
// once the data block is disordered, we do NOT keep last timestamp any more
if (!pTableCxt->ordered) {

View File

@ -825,18 +825,19 @@ static int32_t allocConnRef(SCliConn* conn, bool update) {
transRemoveExHandle(transGetRefMgt(), conn->refId);
conn->refId = -1;
}
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
exh->handle = conn;
exh->pThrd = conn->hostThrd;
QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch);
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
exh->refId = transAddExHandle(transGetRefMgt(), exh);
SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId);
ASSERT(exh == self);
if (self != exh) {
taosMemoryFree(exh);
return TSDB_CODE_REF_INVALID_ID;
}
QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch);
exh->handle = conn;
exh->pThrd = conn->hostThrd;
conn->refId = exh->refId;
if (conn->refId == -1) {
@ -2836,6 +2837,10 @@ int64_t transAllocHandle() {
exh->refId = transAddExHandle(transGetRefMgt(), exh);
SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId);
ASSERT(exh == self);
if (exh != self) {
taosMemoryFree(exh);
return TSDB_CODE_REF_INVALID_ID;
}
QUEUE_INIT(&exh->q);
taosInitRWLatch(&exh->latch);

View File

@ -435,6 +435,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE, "Table already exists in other stables")
TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INCONSISTENT_DB_ID, "Inconsistent database id")
// query
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle")

View File

@ -49,8 +49,7 @@ int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t
int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) {
if (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
@ -58,8 +57,7 @@ int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double num
int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean) {
if (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
@ -67,8 +65,7 @@ int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean
int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) {
if (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
@ -87,8 +84,7 @@ int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem) {
return TSDB_CODE_SUCCESS;
}
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) {
@ -96,8 +92,7 @@ int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) {
return TSDB_CODE_SUCCESS;
}
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj) {
@ -106,18 +101,27 @@ int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void
}
SJson* pJobj = tjsonCreateObject();
if (NULL == pJobj || TSDB_CODE_SUCCESS != func(pObj, pJobj)) {
if (NULL == pJobj) {
return terrno;
}
int32_t rc = func(pObj, pJobj);
if (rc != TSDB_CODE_SUCCESS) {
tjsonDelete(pJobj);
return TSDB_CODE_FAILED;
return rc;
}
return tjsonAddItemToObject(pJson, pName, pJobj);
}
int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) {
SJson* pJobj = tjsonCreateObject();
if (NULL == pJobj || TSDB_CODE_SUCCESS != func(pObj, pJobj)) {
if (pJobj == NULL) {
return terrno;
}
int32_t rc = func(pObj, pJobj);
if (rc != TSDB_CODE_SUCCESS) {
tjsonDelete(pJobj);
return TSDB_CODE_FAILED;
return rc;
}
return tjsonAddItemToArray(pJson, pJobj);
}
@ -156,9 +160,21 @@ int32_t tjsonAddTArray(SJson* pJson, const char* pName, FToJson func, const SArr
return TSDB_CODE_SUCCESS;
}
char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); }
char* tjsonToString(const SJson* pJson) {
char* p = cJSON_Print((cJSON*)pJson);
if (!p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
}
char* tjsonToUnformattedString(const SJson* pJson) { return cJSON_PrintUnformatted((cJSON*)pJson); }
char* tjsonToUnformattedString(const SJson* pJson) {
char* p = cJSON_PrintUnformatted((cJSON*)pJson);
if (!p) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
return p;
}
SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName) { return cJSON_GetObjectItem(pJson, pName); }

View File

@ -27,7 +27,7 @@
,,y,army,./pytest.sh python3 ./test.py -f insert/insert_basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f cluster/splitVgroupByLearner.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f authorith/authBasic.py -N 3
,,n,army,python3 ./test.py -f cmdline/fullopt.py
# ,,n,army,python3 ./test.py -f cmdline/fullopt.py
,,n,army,python3 ./test.py -f query/show.py -N 3
,,n,army,python3 ./test.py -f alter/alterConfig.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f query/subquery/subqueryBugs.py -N 3
@ -278,8 +278,8 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3
#,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1
#,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3
#,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1
,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3

View File

@ -45,7 +45,7 @@ class TDTestCase:
self.TIMESTAMP_BASE = 1706716800
tdSql.init(conn.cursor())
tdSql.execute(f'drop database if exists db')
tdSql.execute(f'create database if not exists db vgroups 1')
tdSql.execute(f'create database if not exists db vgroups 1 keep 10512000m')
tdLog.printNoPrefix("create table")
self.__create_tb()
@ -516,9 +516,19 @@ class TDTestCase:
# check json
self.__insert_query_json("db", "stb_js", "ctb_js", OK_JS, KO_JS, "\'{\"k1\":\"v1\",\"k2\":\"v2\"}\'")
def __insert_query_ts5184(self, dbname="db", stbname="stb_ts5184", ctbname="ctb_ts5184", ntbname="ntb_ts5184"):
TB_LIST = [ ctbname, ntbname]
tdSql.execute(f'create table {dbname}.{stbname} (ts timestamp, w_ts timestamp, opc nchar(100),quality int) tags(t0 int);')
tdSql.execute(f'create table {dbname}.{ntbname} (ts timestamp, w_ts timestamp, opc nchar(100),quality int);')
tdSql.execute(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(1);')
for _tb in TB_LIST:
tdSql.execute(f'insert into {dbname}.{_tb} values(1721265436000,now(),"0",192) {dbname}.{_tb}(quality,w_ts,ts) values(192,now(),1721265326000) {dbname}.{_tb}(quality,w_ts,ts) values(190,now()+1s,1721265326000) {dbname}.{_tb} values(1721265436000,now()+2s,"1",191) {dbname}.{_tb}(quality,w_ts,ts) values(192,now()+3s,1721265326002) {dbname}.{_tb}(ts,w_ts,opc,quality) values(1721265436003,now()+4s,"3",193);');
tdSql.query(f'select * from {dbname}.{_tb}', show=True)
tdSql.checkRows(4)
def run(self):
self.__insert_query_exec()
self.__insert_query_ts5184()
def stop(self):