diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index e23ebb104b..89d1066a43 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 11b60a4 + GIT_TAG 4efbc10 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/include/client/taos.h b/include/client/taos.h index e54300e33e..838d0e8266 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -220,6 +220,7 @@ DLL_EXPORT const void *taos_get_raw_block(TAOS_RES *res); DLL_EXPORT int taos_get_db_route_info(TAOS *taos, const char *db, TAOS_DB_ROUTE_INFO *dbInfo); DLL_EXPORT int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId); +DLL_EXPORT int taos_get_tables_vgId(TAOS *taos, const char *db, const char *table[], int tableNum, int *vgId); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 6350057c1f..b7061300c1 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -342,8 +342,8 @@ typedef struct tDataTypeDescriptor { extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX]; bool isValidDataType(int32_t type); +int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type); -void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); void *getDataMin(int32_t type, void* value); void *getDataMax(int32_t type, void* value); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 0e2d043174..fbb24d2862 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -210,6 +210,9 @@ int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STabl int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetTablesHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctId, const char* pDb, const char* pTableName[], + int32_t tableNum, int32_t *vgId); + int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); int32_t catalogGetCachedTableVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index cfd5bd1ed7..63e1c556de 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -154,6 +154,8 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo); */ int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode); +bool qTaskIsExecuting(qTaskInfo_t qinfo); + /** * destroy query info structure * @param qHandle diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index d37f8f76c2..f93bf9a326 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -232,6 +232,7 @@ int64_t syncOpen(SSyncInfo* pSyncInfo); int32_t syncStart(int64_t rid); void syncStop(int64_t rid); void syncPreStop(int64_t rid); +void syncPostStop(int64_t rid); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b315432be1..51f336c3aa 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -345,6 +345,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOPIC_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x03EB) #define TSDB_CODE_MND_CGROUP_USED TAOS_DEF_ERROR_CODE(0, 0x03EC) #define TSDB_CODE_MND_TOPIC_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03ED) +#define TSDB_CODE_MND_INVALID_SUB_OPTION TAOS_DEF_ERROR_CODE(0, 0x03EE) #define TSDB_CODE_MND_IN_REBALANCE TAOS_DEF_ERROR_CODE(0, 0x03EF) // mnode-stream diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index e37ed4c1bc..7f79323c4c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1168,6 +1168,54 @@ _return: return code; } +int taos_get_tables_vgId(TAOS *taos, const char *db, const char *table[], int tableNum, int *vgId) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return terrno; + } + + if (NULL == db || NULL == table || NULL == vgId || tableNum <= 0) { + tscError("invalid input param, db:%p, table:%p, vgId:%p, tbNum:%d", db, table, vgId, tableNum); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + return terrno; + } + + int64_t connId = *(int64_t *)taos; + SRequestObj *pRequest = NULL; + char *sql = "taos_get_table_vgId"; + int32_t code = buildRequest(connId, sql, strlen(sql), NULL, false, &pRequest, 0); + if (code != TSDB_CODE_SUCCESS) { + return terrno; + } + + pRequest->syncQuery = true; + + STscObj *pTscObj = pRequest->pTscObj; + SCatalog *pCtg = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg); + if (code != TSDB_CODE_SUCCESS) { + goto _return; + } + + SRequestConnInfo conn = { + .pTrans = pTscObj->pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; + + conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + code = catalogGetTablesHashVgId(pCtg, &conn, pTscObj->acctId, db, table, tableNum, vgId); + if (code) { + goto _return; + } + +_return: + + terrno = code; + + destroyRequest(pRequest); + return code; +} + + int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index bdba83cbb2..90c405c204 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -912,10 +912,12 @@ void tmqFreeImpl(void* handle) { tmq_t* tmq = (tmq_t*)handle; // TODO stop timer - tmqClearUnhandleMsg(tmq); - if (tmq->mqueue) taosCloseQueue(tmq->mqueue); + if (tmq->mqueue) { + tmqClearUnhandleMsg(tmq); + taosCloseQueue(tmq->mqueue); + } if (tmq->delayedTask) taosCloseQueue(tmq->delayedTask); - if (tmq->qall) taosFreeQall(tmq->qall); + taosFreeQall(tmq->qall); tsem_destroy(&tmq->rspSem); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2b43229b83..171679baae 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -22,7 +22,6 @@ #define MALLOC_ALIGN_BYTES 256 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { - ASSERT(pColumnInfoData != NULL); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length; } else { @@ -65,8 +64,6 @@ int32_t getJsonValueLen(const char* data) { } int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) { - ASSERT(pColumnInfoData != NULL); - if (isNull) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -111,7 +108,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con uint32_t len = pColumnInfoData->varmeta.length; pColumnInfoData->varmeta.offset[currentRow] = len; - memcpy(pColumnInfoData->pData + len, pData, dataLen); + memmove(pColumnInfoData->pData + len, pData, dataLen); pColumnInfoData->varmeta.length += dataLen; } else { memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); @@ -177,8 +174,6 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows) { - ASSERT(pData != NULL && pColumnInfoData != NULL); - int32_t len = pColumnInfoData->info.bytes; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { len = varDataTLen(pData); @@ -236,7 +231,10 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2) { - ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); + if (pColumnInfoData->info.type != pSource->info.type) { + return TSDB_CODE_FAILED; + } + if (numOfRow2 == 0) { return numOfRow1; } @@ -316,13 +314,13 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) { - ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); - if (numOfRows <= 0) { - return numOfRows; + if (pColumnInfoData->info.type != pSource->info.type || + (pBlockInfo != NULL && pBlockInfo->capacity < numOfRows)) { + return TSDB_CODE_FAILED; } - if (pBlockInfo != NULL) { - ASSERT(pBlockInfo->capacity >= numOfRows); + if (numOfRows <= 0) { + return numOfRows; } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -388,7 +386,6 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) } int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { - assert(pSrc != NULL && pDest != NULL); int32_t capacity = pDest->info.capacity; size_t numOfCols = taosArrayGetSize(pDest->pDataBlock); @@ -406,8 +403,6 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { } size_t blockDataGetSize(const SSDataBlock* pBlock) { - assert(pBlock != NULL); - size_t total = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { @@ -422,8 +417,6 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) { // Actual data rows pluses the corresponding meta data must fit in one memory buffer of the given page size. int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize) { - ASSERT(pBlock != NULL && stopIndex != NULL); - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; @@ -437,7 +430,9 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd if (!hasVarCol) { size_t rowSize = blockDataGetRowSize(pBlock); int32_t capacity = payloadSize / (rowSize + numOfCols * bitmapChar / 8.0); - ASSERT(capacity > 0); + if (capacity <= 0) { + return TSDB_CODE_FAILED; + } *stopIndex = startIndex + capacity - 1; if (*stopIndex >= numOfRows) { @@ -469,7 +464,9 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd if (size > pageSize) { // pageSize must be able to hold one row *stopIndex = j - 1; - ASSERT(*stopIndex >= startIndex); + if (*stopIndex < startIndex) { + return TSDB_CODE_FAILED; + } return TSDB_CODE_SUCCESS; } @@ -540,8 +537,6 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 * @return */ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { - ASSERT(pBlock != NULL); - // write the number of rows *(uint32_t*)buf = pBlock->info.rows; @@ -612,7 +607,9 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { } pCol->varmeta.length = colLength; - ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen); + if (pCol->varmeta.length > pCol->varmeta.allocLen) { + return TSDB_CODE_FAILED; + } } memcpy(pCol->pData, pStart, colLength); @@ -659,7 +656,9 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) } pCol->varmeta.length = colLength; - ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen); + if (pCol->varmeta.length > pCol->varmeta.allocLen) { + return TSDB_CODE_FAILED; + } } if (!colDataIsNNull_s(pCol, 0, pBlock->info.rows)) { @@ -673,7 +672,6 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) } size_t blockDataGetRowSize(SSDataBlock* pBlock) { - ASSERT(pBlock != NULL); if (pBlock->info.rowSize == 0) { size_t rowSize = 0; @@ -702,7 +700,6 @@ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { - ASSERT(pBlock != NULL); double rowSize = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); @@ -905,7 +902,6 @@ static int32_t* createTupleIndex(size_t rows) { static void destroyTupleIndex(int32_t* index) { taosMemoryFreeClear(index); } int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { - ASSERT(pDataBlock != NULL && pOrderInfo != NULL); if (pDataBlock->info.rows <= 1) { return TSDB_CODE_SUCCESS; } @@ -1149,8 +1145,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { void blockDataEmpty(SSDataBlock* pDataBlock) { SDataBlockInfo* pInfo = &pDataBlock->info; - ASSERT(pInfo->rows <= pDataBlock->info.capacity); - if (pInfo->capacity == 0) { + if (pInfo->capacity == 0 || pInfo->rows > pDataBlock->info.capacity) { return; } @@ -1168,8 +1163,7 @@ void blockDataEmpty(SSDataBlock* pDataBlock) { // todo temporarily disable it static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) { - ASSERT(numOfRows > 0); - if (numOfRows <= pBlockInfo->capacity) { + if (numOfRows <= 0 || numOfRows <= pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } @@ -1196,7 +1190,9 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); - ASSERT(pColumn->info.bytes); + if (pColumn->info.bytes == 0) { + return TSDB_CODE_FAILED; + } // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned tmp = taosMemoryMallocAlign(MALLOC_ALIGN_BYTES, numOfRows * pColumn->info.bytes); @@ -1214,7 +1210,9 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* // todo remove it soon #if defined LINUX - ASSERT((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) == 0x0); + if ((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) != 0x0) { + return TSDB_CODE_FAILED; + } #endif if (clearPayload) { @@ -1308,8 +1306,6 @@ void* blockDataDestroy(SSDataBlock* pBlock) { } int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { - ASSERT(src != NULL); - dst->info = src->info; dst->info.rows = 0; dst->info.capacity = 0; @@ -1344,8 +1340,6 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { } int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) { - ASSERT(src != NULL && dst != NULL); - blockDataCleanup(dst); int32_t code = blockDataEnsureCapacity(dst, src->info.rows); if (code != TSDB_CODE_SUCCESS) { @@ -1501,7 +1495,6 @@ SSDataBlock* createDataBlock() { } int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData) { - ASSERT(pBlock != NULL && pColInfoData != NULL); if (pBlock->pDataBlock == NULL) { pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); if (pBlock->pDataBlock == NULL) { @@ -1536,7 +1529,6 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) } SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index) { - ASSERT(pBlock != NULL); if (index >= taosArrayGetSize(pBlock->pDataBlock)) { return NULL; } @@ -2142,7 +2134,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_MEDIUMBLOB: uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); - ASSERT(0); break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { @@ -2176,7 +2167,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB } } else { uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - ASSERT(0); } break; } @@ -2222,7 +2212,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB } char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { - ASSERT(stbFullName[0] != 0); + if (stbFullName[0] == 0) { + return NULL; + } + SArray* tags = taosArrayInit(0, sizeof(void*)); if (tags == NULL) { return NULL; @@ -2260,7 +2253,9 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { taosMemoryFree(pTag); taosArrayDestroy(tags); - ASSERT(rname.ctbShortName && rname.ctbShortName[0]); + if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { + return NULL; + } return rname.ctbShortName; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 0d47ef1e7f..5cb3fe4dc0 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -98,8 +98,6 @@ SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName } int32_t tNameExtractFullName(const SName* name, char* dst) { - assert(name != NULL && dst != NULL); - // invalid full name format, abort if (!tNameIsValid(name)) { return -1; @@ -109,7 +107,7 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { size_t tnameLen = strlen(name->tname); if (tnameLen > 0) { - /*assert(name->type == TSDB_TABLE_NAME_T);*/ + /*ASSERT(name->type == TSDB_TABLE_NAME_T);*/ dst[len] = TS_PATH_DELIMITER[0]; memcpy(dst + len + 1, name->tname, tnameLen); @@ -120,25 +118,21 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { } int32_t tNameLen(const SName* name) { - assert(name != NULL); - char tmp[12] = {0}; int32_t len = sprintf(tmp, "%d", name->acctId); int32_t len1 = (int32_t)strlen(name->dbname); int32_t len2 = (int32_t)strlen(name->tname); if (name->type == TSDB_DB_NAME_T) { - assert(len2 == 0); + ASSERT(len2 == 0); return len + len1 + TSDB_NAME_DELIMITER_LEN; } else { - assert(len2 > 0); + ASSERT(len2 > 0); return len + len1 + len2 + TSDB_NAME_DELIMITER_LEN * 2; } } bool tNameIsValid(const SName* name) { - assert(name != NULL); - if (!VALID_NAME_TYPE(name->type)) { return false; } @@ -151,15 +145,12 @@ bool tNameIsValid(const SName* name) { } SName* tNameDup(const SName* name) { - assert(name != NULL); - SName* p = taosMemoryMalloc(sizeof(SName)); memcpy(p, name, sizeof(SName)); return p; } int32_t tNameGetDbName(const SName* name, char* dst) { - assert(name != NULL && dst != NULL); strncpy(dst, name->dbname, tListLen(name->dbname)); return 0; } @@ -167,28 +158,24 @@ int32_t tNameGetDbName(const SName* name, char* dst) { const char* tNameGetDbNameP(const SName* name) { return &name->dbname[0]; } int32_t tNameGetFullDbName(const SName* name, char* dst) { - assert(name != NULL && dst != NULL); snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); return 0; } bool tNameIsEmpty(const SName* name) { - assert(name != NULL); return name->type == 0 || name->acctId == 0; } const char* tNameGetTableName(const SName* name) { - assert(name != NULL && name->type == TSDB_TABLE_NAME_T); + ASSERT(name != NULL && name->type == TSDB_TABLE_NAME_T); return &name->tname[0]; } void tNameAssign(SName* dst, const SName* src) { memcpy(dst, src, sizeof(SName)); } int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t nameLen) { - assert(dst != NULL && dbName != NULL && nameLen > 0); - // too long account id or too long db name - if (nameLen >= tListLen(dst->dbname)) { + if (nameLen <= 0 || nameLen >= tListLen(dst->dbname)) { return -1; } @@ -199,8 +186,6 @@ int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t name } int32_t tNameAddTbName(SName* dst, const char* tbName, size_t nameLen) { - assert(dst != NULL && tbName != NULL && nameLen > 0); - // too long account id or too long db name if (nameLen >= tListLen(dst->tname) || nameLen <= 0) { return -1; @@ -212,7 +197,6 @@ int32_t tNameAddTbName(SName* dst, const char* tbName, size_t nameLen) { } int32_t tNameSetAcctId(SName* dst, int32_t acctId) { - assert(dst != NULL); dst->acctId = acctId; return 0; } @@ -247,7 +231,9 @@ bool tNameTbNameEqual(SName* left, SName* right) { } int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { - assert(dst != NULL && str != NULL && strlen(str) > 0); + if (strlen(str) == 0) { + return -1; + } char* p = NULL; if ((type & T_NAME_ACCT) == T_NAME_ACCT) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index ca2c056743..969da4226f 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -76,7 +76,6 @@ void tdSCellValPrint(SCellVal *pVal, int8_t colType) { return; } if (!pVal->val) { - ASSERT(0); printf("BadVal "); return; } @@ -490,7 +489,6 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -512,7 +510,6 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa *pValType = ((*pDestByte) & 0x03); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -521,7 +518,6 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -555,7 +551,6 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal *pValType = ((*pDestByte) & 0x01); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -564,7 +559,6 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -607,7 +601,6 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { // *pDestByte |= (valType); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -630,7 +623,6 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ output->val = POINTER_SHIFT(pRow, offset); } #else - ASSERT(0); if (offset < 0) { terrno = TSDB_CODE_INVALID_PARA; output->valType = TD_VTYPE_NONE; @@ -680,7 +672,6 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp return terrno; } #else - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; #endif @@ -707,8 +698,8 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp if (!pBuilder->hasNone) pBuilder->hasNone = true; return TSDB_CODE_SUCCESS; default: - ASSERT(0); - break; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } if (TD_IS_TP_ROW(pRow)) { @@ -722,7 +713,6 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -810,7 +800,6 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou pBuilder->nCols = nCols; pBuilder->nBoundCols = nBoundCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -832,7 +821,6 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -869,7 +857,6 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -880,7 +867,6 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -900,7 +886,6 @@ int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { #endif break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -920,7 +905,6 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { pBuilder->flen = flen; pBuilder->nCols = nCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -939,7 +923,6 @@ int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, pBuilder->nCols = nCols; pBuilder->nBoundCols = nBoundCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -968,7 +951,6 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT tdGetBitmapValTypeI(pBitmap, colIdx, pValType); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; } @@ -987,7 +969,6 @@ bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1014,7 +995,6 @@ int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { // *pDestByte |= (valType); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1031,7 +1011,6 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int tdSetBitmapValTypeI(pBitmap, colIdx, valType); break; default: - ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index ec05ef4c44..559ffd2aaf 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -168,12 +168,13 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) { i = MICRO_SEC_FRACTION_LEN; } times = MICRO_SEC_FRACTION_LEN - i; - } else { - assert(timePrec == TSDB_TIME_PRECISION_NANO); + } else if (timePrec == TSDB_TIME_PRECISION_NANO) { if (i >= NANO_SEC_FRACTION_LEN) { i = NANO_SEC_FRACTION_LEN; } times = NANO_SEC_FRACTION_LEN - i; + } else { + return -1; } fraction = strnatoi(str, i) * factor[times]; @@ -510,8 +511,11 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre // !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = // 1626006833631000064 int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { - assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || - fromPrecision == TSDB_TIME_PRECISION_NANO); + if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO && + fromPrecision != TSDB_TIME_PRECISION_NANO) { + return -1; + } + int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; double tmp = time; switch (toUnit) { @@ -761,8 +765,7 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char } int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { - if (pInterval->sliding == 0) { - assert(pInterval->interval == 0); + if (pInterval->sliding == 0 && pInterval->interval == 0) { return t; } @@ -931,7 +934,7 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) default: fractionLen = 0; - assert(false); + ASSERT(false); } taosLocalTime(", &ptm); diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 3a6bc097c1..0faa6eb4c8 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -17,6 +17,7 @@ #include "ttszip.h" #include "taoserror.h" #include "tcompression.h" +#include "tlog.h" static int32_t getDataStartOffset(); static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); @@ -202,14 +203,14 @@ void* tsBufDestroy(STSBuf* pTSBuf) { static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) { int32_t last = pTSBuf->numOfGroups - 1; - assert(last >= 0); + ASSERT(last >= 0); return &pTSBuf->pData[last]; } static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) { uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); - assert((int32_t)newSize > pTSBuf->numOfAlloc); + ASSERT((int32_t)newSize > pTSBuf->numOfAlloc); STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); @@ -233,7 +234,7 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info; pBlockInfo->id = id; pBlockInfo->offset = pTSBuf->fileSize; - assert(pBlockInfo->offset >= getDataStartOffset()); + ASSERT(pBlockInfo->offset >= getDataStartOffset()); // update vnode info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo); @@ -282,7 +283,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { pTsData->allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET); - assert(r == 0); + ASSERT(r == 0); /* * format for output data: @@ -316,7 +317,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); - assert(metaLen == getTagAreaLength(&pBlock->tag)); + ASSERT(metaLen == getTagAreaLength(&pBlock->tag)); int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; @@ -379,7 +380,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { size_t sz = 0; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { char* tp = taosMemoryRealloc(pBlock->tag.pz, pBlock->tag.nLen + 1); - assert(tp != NULL); + ASSERT(tp != NULL); memset(tp, 0, pBlock->tag.nLen + 1); pBlock->tag.pz = tp; @@ -410,14 +411,14 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // read the comp length at the length of comp block sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); - assert(pBlock->padding == pBlock->compLen); + ASSERT(pBlock->padding == pBlock->compLen); int32_t n = 0; sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen)); if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { - assert(n == 0); + ASSERT(n == 0); } else { - assert(n == pBlock->tag.nLen); + ASSERT(n == pBlock->tag.nLen); } UNUSED(sz); @@ -477,7 +478,7 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, i pBlockInfo = tsBufGetLastGroupInfo(pTSBuf); } - assert(pBlockInfo->info.id == id); + ASSERT(pBlockInfo->info.id == id); if ((taosVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first @@ -596,7 +597,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) { STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; if (pBlockInfo->numOfBlocks <= blockIndex) { - assert(false); + ASSERT(false); } STSCursor* pCur = &pTSBuf->cur; @@ -613,7 +614,7 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex } } else { if (tsBufFindBlock(pTSBuf, pBlockInfo, blockIndex) == -1) { - assert(false); + ASSERT(false); } } @@ -633,7 +634,7 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); + ASSERT((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); pCur->vgroupIndex = groupIndex; pCur->blockIndex = blockIndex; @@ -668,7 +669,9 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { return -1; } - assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); + if (pHeader->tsOrder != TSDB_ORDER_ASC && pHeader->tsOrder != TSDB_ORDER_DESC) { + return -1; + } int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); if (r != 0) { @@ -705,7 +708,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { } } else { // get the last timestamp record in the last block of the last vnode - assert(pTSBuf->numOfGroups > 0); + ASSERT(pTSBuf->numOfGroups > 0); int32_t groupIndex = pTSBuf->numOfGroups - 1; pCur->vgroupIndex = groupIndex; @@ -729,7 +732,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1; while (1) { - assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); + ASSERT(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { @@ -810,7 +813,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { } // src can only have one vnode index - assert(pSrcBuf->numOfGroups == 1); + ASSERT(pSrcBuf->numOfGroups == 1); // there are data in buffer, flush to disk first tsBufFlush(pDestBuf); @@ -853,7 +856,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { } int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END); - assert(r == 0); + ASSERT(r == 0); int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; @@ -881,7 +884,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { } pDestBuf->fileSize = (uint32_t)file_size; - assert(pDestBuf->fileSize == oldSize + size); + ASSERT(pDestBuf->fileSize == oldSize + size); return 0; } @@ -913,7 +916,10 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ pTSBuf->fileSize += len; pTSBuf->tsOrder = order; - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + if (order != TSDB_ORDER_ASC && order != TSDB_ORDER_DESC) { + tsBufDestroy(pTSBuf); + return NULL; + } STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; @@ -1095,7 +1101,7 @@ void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { } int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { - assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); + ASSERT(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; *len = 0; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index d412fd89da..c83bdc0e32 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -140,7 +140,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { } } -void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { +int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { if (optr == OP_TYPE_ADD) { switch (type) { case TSDB_DATA_TYPE_TINYINT: @@ -177,11 +177,12 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { SET_DOUBLE_VAL(dst, GET_DOUBLE_VAL(s1) + GET_DOUBLE_VAL(s2)); break; default: { - assert(0); - break; + return -1; } } } else { - assert(0); + return -1; } + + return 0; } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 65e9a767f5..de225581a6 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -168,7 +168,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { pSrc->nType == TSDB_DATA_TYPE_JSON) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; char *p = taosMemoryRealloc(pDst->pz, len); - assert(p); + ASSERT(p); memset(p, 0, len); pDst->pz = p; @@ -192,7 +192,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { size_t num = taosArrayGetSize(pSrc->arr); pDst->arr = taosArrayInit(num, sizeof(int64_t)); pDst->nLen = pSrc->nLen; - assert(pSrc->nLen == num); + ASSERT(pSrc->nLen == num); for (size_t i = 0; i < num; i++) { int64_t *p = taosArrayGet(pSrc->arr, i); taosArrayPush(pDst->arr, p); diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 9bd5be5201..e8402eb7c0 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -58,11 +58,7 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { dTrace("msg:%p, get from snode-stream queue", pMsg); int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg); if (code < 0) { - if (pMsg) { - dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code)); - } else { - dGError("snd, msg:%p failed to process stream empty msg since %s", pMsg, terrstr(code)); - } + dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code)); smSendRsp(pMsg, terrno); } @@ -161,8 +157,10 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { smPutNodeMsgToWriteQueue(pMgmt, pMsg); break; default: - ASSERTS(0, "msg:%p failed to put into snode queue since %s, type:%s qtype:%d", pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), qtype); + terrno = TSDB_CODE_INVALID_PARA; + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + return -1; } return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 39733c5a5a..3ce37a5f8e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -132,7 +132,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->syncCfg.myIndex = pCreate->selfIndex; pCfg->syncCfg.replicaNum = pCreate->replica; memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo)); - for (int i = 0; i < pCreate->replica; ++i) { + for (int32_t i = 0; i < pCreate->replica; ++i) { SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; pNode->nodeId = pCreate->replicas[i].id; pNode->nodePort = pCreate->replicas[i].port; @@ -288,7 +288,8 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("vgId:%d, start to alter vnode, replica:%d selfIndex:%d strict:%d", alterReq.vgId, alterReq.replica, alterReq.selfIndex, alterReq.strict); for (int32_t i = 0; i < alterReq.replica; ++i) { - dInfo("vgId:%d, replica:%d ep:%s:%u", alterReq.vgId, i, alterReq.replicas[i].fqdn, alterReq.replicas[i].port); + SReplica *pReplica = &alterReq.replicas[i]; + dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", alterReq.vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); } if (alterReq.replica <= 0 || alterReq.selfIndex < 0 || alterReq.selfIndex >= alterReq.replica) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index f808c67ef6..6b274261e5 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -118,6 +118,9 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { dInfo("vgId:%d, all vnode queues is empty", pVnode->vgId); + dInfo("vgId:%d, post close", pVnode->vgId); + vnodePostClose(pVnode->pImpl); + vmFreeQueue(pMgmt, pVnode); vnodeClose(pVnode->pImpl); pVnode->pImpl = NULL; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 202dc50ac6..cd29b11550 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -86,12 +86,8 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - if (pMsg) { - dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType), - terrstr(code)); - } else { - dGError("vgId:%d, msg:%p failed to process stream empty msg since %s", pVnode->vgId, pMsg, terrstr(code)); - } + dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType), + terrstr(code)); vmSendRsp(pMsg, code); } @@ -138,6 +134,13 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf } } +static void vmSendResponse(SRpcMsg *pMsg) { + if (pMsg->info.handle) { + SRpcMsg rsp = {.info = pMsg->info, .code = terrno}; + rpcSendResponse(&rsp); + } +} + static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { const STraceId *trace = &pMsg->info.traceId; if (pMsg->contLen < sizeof(SMsgHead)) { @@ -146,17 +149,19 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp return -1; } - SMsgHead *pHead = pMsg->pCont; - int32_t code = 0; + SMsgHead *pHead = pMsg->pCont; + int32_t code = 0; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); - return terrno != 0 ? terrno : -1; + dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, + terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); + terrno = (terrno != 0) ? terrno : -1; + vmSendResponse(pMsg); + return terrno; } switch (qtype) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 8a4f6c2195..87b5a5c42d 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -112,7 +112,6 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER) for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { - ASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions); SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i); SDB_SET_INT64(pRaw, dataPos, pRetension->freq, _OVER) SDB_SET_INT64(pRaw, dataPos, pRetension->keep, _OVER) @@ -364,6 +363,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1; if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1; if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; + if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1; terrno = 0; return terrno; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index b5c2fb05b3..a5f77513de 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -489,7 +489,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { tlen += tEncodeSMqConsumerEp(buf, pConsumerEp); cnt++; } - ASSERT(cnt == sz); + if(cnt != sz) return -1; tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp); tlen += taosEncodeString(buf, pSub->dbName); return tlen; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d7b16c2c8e..c7a416d444 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1050,7 +1050,7 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB status = "offline"; } - char b1[9] = {0}; + char b1[16] = {0}; STR_TO_VARSTR(b1, status); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 37fc9f79e1..c8c8e06c5e 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -763,7 +763,6 @@ static void mndReloadSyncConfig(SMnode *pMnode) { mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); return; } - // ASSERT(0); if (cfg.myIndex == -1) { #if 1 diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 6a7e2aaa51..7a8de4099f 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -111,7 +111,7 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { } else if (strncasecmp(name, TSDB_INS_TABLE_USER_PRIVILEGES, len) == 0) { type = TSDB_MGMT_TABLE_PRIVILEGES; } else { - // ASSERT(0); + mError("invalid show name:%s len:%d", name, len); } return type; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 2d1f6eb505..141bb1df60 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -488,7 +488,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN); smaObj.createdTime = taosGetTimestampMs(); smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); - ASSERT(smaObj.uid != 0); + char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0}; snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "%s_td_tsma_rst_tb", pCreate->name); memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN); @@ -558,13 +558,15 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea SNode *pAst = NULL; if (nodesStringToNode(streamObj.ast, &pAst) < 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since parse ast error", smaObj.name); return -1; } // extract output schema from ast if (qExtractResultSchema(pAst, (int32_t *)&streamObj.outputSchema.nCols, &streamObj.outputSchema.pSchema) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since extract result schema error", smaObj.name); return -1; } @@ -579,15 +581,18 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea }; if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since create query plan error", smaObj.name); return -1; } // save physcial plan if (nodesNodeToString((SNode *)pPlan, false, &streamObj.physicalPlan, NULL) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since save physcial plan error", smaObj.name); return -1; } + if (pAst != NULL) nodesDestroyNode(pAst); nodesDestroyNode((SNode *)pPlan); @@ -826,14 +831,13 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); sdbRelease(pMnode->pSdb, pStream); - ASSERT(0); goto _OVER; } // drop stream if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + mError("stream:%s, failed to drop log since %s", pStream->name, terrstr()); sdbRelease(pMnode->pSdb, pStream); - ASSERT(0); goto _OVER; } } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 54b3202d24..d504a94700 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1177,7 +1177,9 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, SNode *pAst = NULL; if (nodesStringToNode(pTopic->ast, &pAst) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; + mError("topic:%s, create ast error", pTopic->name); + sdbRelease(pSdb, pTopic); return -1; } @@ -1222,7 +1224,9 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName SNode *pAst = NULL; if (nodesStringToNode(pStream->ast, &pAst) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; + mError("stream:%s, create ast error", pStream->name); + sdbRelease(pSdb, pStream); return -1; } @@ -2094,7 +2098,9 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, SNode *pAst = NULL; if (nodesStringToNode(pTopic->ast, &pAst) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + mError("topic:%s, create ast error", pTopic->name); + sdbRelease(pSdb, pTopic); return -1; } @@ -2141,7 +2147,9 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, SNode *pAst = NULL; if (nodesStringToNode(pStream->ast, &pAst) != 0) { - ASSERT(0); + terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; + mError("stream:%s, create ast error", pStream->name); + sdbRelease(pSdb, pStream); return -1; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 418474baa6..b8ef185199 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -96,18 +96,12 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, pSub->subType = pTopic->subType; pSub->withMeta = pTopic->withMeta; - ASSERT(pSub->unassignedVgs->size == 0); - ASSERT(taosHashGetSize(pSub->consumerHash) == 0); - if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { tDeleteSubscribeObj(pSub); taosMemoryFree(pSub); return NULL; } - ASSERT(pSub->unassignedVgs->size > 0); - ASSERT(taosHashGetSize(pSub->consumerHash) == 0); - return pSub; } @@ -144,7 +138,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg) { - ASSERT(pRebVg->oldConsumerId != pRebVg->newConsumerId); + if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { + terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; + return -1; + } void *buf; int32_t tlen; @@ -155,8 +152,8 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM int32_t vgId = pRebVg->pVgEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); if (pVgObj == NULL) { - ASSERT(0); taosMemoryFree(buf); + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -206,9 +203,9 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { } static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { - int32_t totalVgNum = pOutput->pSub->vgNum; - - mInfo("mq rebalance: subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); + int32_t totalVgNum = pOutput->pSub->vgNum; + const char *sub = pOutput->pSub->key; + mInfo("sub:%s, mq rebalance vgNum:%d", sub, pOutput->pSub->vgNum); // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -218,11 +215,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR int32_t actualRemoved = 0; for (int32_t i = 0; i < removedNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); - ASSERT(consumerId > 0); + SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); - ASSERT(pConsumerEp); + if (pConsumerEp) { - ASSERT(consumerId == pConsumerEp->consumerId); actualRemoved++; int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); for (int32_t j = 0; j < consumerVgNum; j++) { @@ -233,7 +229,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64, pVgEp->vgId, consumerId); + mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64, sub, pVgEp->vgId, consumerId); } taosArrayDestroy(pConsumerEp->vgs); taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); @@ -241,7 +237,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pOutput->removedConsumers, &consumerId); } } - ASSERT(removedNum == actualRemoved); + + if (removedNum != actualRemoved) { + mError("sub:%s, mq rebalance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved); + } // if previously no consumer, there are vgs not assigned { @@ -254,7 +253,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from unassigned", pVgEp->vgId); + mInfo("sub:%s, mq rebalance remove vgId:%d from unassigned", sub, pVgEp->vgId); } } @@ -268,8 +267,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR minVgCnt = totalVgNum / afterRebConsumerNum; imbConsumerNum = totalVgNum % afterRebConsumerNum; } - mInfo("mq rebalance: %d consumer after rebalance, at least %d vg each, %d consumer has more vg", afterRebConsumerNum, - minVgCnt, imbConsumerNum); + mInfo("sub:%s, mq rebalance %d consumer after rebalance, at least %d vg each, %d consumer has more vg", sub, + afterRebConsumerNum, minVgCnt, imbConsumerNum); // 4. first scan: remove consumer more than wanted, put to remove hash int32_t imbCnt = 0; @@ -278,7 +277,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - ASSERT(pConsumerEp->consumerId > 0); + int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); // all old consumers still existing are touched // TODO optimize: touch only consumer whose vgs changed @@ -298,7 +297,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId, pConsumerEp->consumerId); } imbCnt++; @@ -313,7 +312,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId, pConsumerEp->consumerId); } } @@ -325,13 +324,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers); for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); - ASSERT(consumerId > 0); + SMqConsumerEp newConsumerEp; newConsumerEp.consumerId = consumerId; newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosArrayPush(pOutput->newConsumers, &consumerId); - mInfo("mq rebalance: add new consumer:%" PRId64, consumerId); + mInfo("sub:%s, mq rebalance add new consumer:%" PRId64, sub, consumerId); } } @@ -344,13 +343,16 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; - ASSERT(pConsumerEp->consumerId > 0); // push until equal minVg while (taosArrayGetSize(pConsumerEp->vgs) < minVgCnt) { // iter hash and find one vg pRemovedIter = taosHashIterate(pHash, pRemovedIter); - ASSERT(pRemovedIter); + if (pRemovedIter == NULL) { + mError("sub:%s, removed iter is null", sub); + continue; + } + pRebVg = (SMqRebOutputVg *)pRemovedIter; // push taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); @@ -361,7 +363,6 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } } - ASSERT(pIter == NULL); // 7. handle unassigned vg if (taosHashGetSize(pOutput->pSub->consumerHash) != 0) { // if has consumer, assign all left vg @@ -377,9 +378,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); - ASSERT(pIter); pConsumerEp = (SMqConsumerEp *)pIter; - ASSERT(pConsumerEp->consumerId > 0); + if (taosArrayGetSize(pConsumerEp->vgs) == minVgCnt) { break; } @@ -404,19 +404,19 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR pIter = taosHashIterate(pHash, pIter); if (pIter == NULL) break; pRebOutput = (SMqRebOutputVg *)pIter; - ASSERT(pRebOutput->newConsumerId == -1); + taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->rebVgs, pRebOutput); - mInfo("mq rebalance: unassign vgId:%d (second scan)", pRebOutput->pVgEp->vgId); + mInfo("sub:%s, mq rebalance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId); } } // 8. generate logs - mInfo("mq rebalance: calculation completed, rebalanced vg:"); + mInfo("sub:%s, mq rebalance calculation completed, rebalanced vg", sub); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("mq rebalance: vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId, - pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); + mInfo("sub:%s, mq rebalance vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, sub, + pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } { void *pIter = NULL; @@ -425,10 +425,11 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - mInfo("mq rebalance: final cfg: consumer %" PRId64 " has %d vg", pConsumerEp->consumerId, sz); + mInfo("sub:%s, mq rebalance final cfg: consumer %" PRId64 " has %d vg", sub, pConsumerEp->consumerId, sz); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); - mInfo("mq rebalance: final cfg: vg %d to consumer %" PRId64 "", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("sub:%s, mq rebalance final cfg: vg %d to consumer %" PRId64 "", sub, pVgEp->vgId, + pConsumerEp->consumerId); } } } @@ -487,7 +488,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu consumerNum = taosArrayGetSize(pOutput->newConsumers); for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i); - ASSERT(consumerId > 0); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); pConsumerNew->updateType = CONSUMER_UPDATE__ADD; @@ -497,7 +498,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu taosArrayPush(pConsumerNew->rebNewTopics, &topic); mndReleaseConsumer(pMnode, pConsumerOld); if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { - ASSERT(0); tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); goto REB_FAIL; @@ -510,7 +510,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu consumerNum = taosArrayGetSize(pOutput->removedConsumers); for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i); - ASSERT(consumerId > 0); + SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup); pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE; @@ -520,7 +520,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu taosArrayPush(pConsumerNew->rebRemovedTopics, &topic); mndReleaseConsumer(pMnode, pConsumerOld); if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { - ASSERT(0); tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); goto REB_FAIL; @@ -577,7 +576,6 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { char cgroup[TSDB_CGROUP_LEN]; mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - /*ASSERT(pTopic);*/ if (pTopic == NULL) { mError("mq rebalance %s failed since topic %s not exist, abort", pRebInfo->key, topic); continue; @@ -585,8 +583,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { taosRLockLatch(&pTopic->lock); rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key); + + if (rebOutput.pSub == NULL) { + mError("mq rebalance %s failed create sub since %s, abort", pRebInfo->key, terrstr()); + taosRUnLockLatch(&pTopic->lock); + mndReleaseTopic(pMnode, pTopic); + continue; + } memcpy(rebOutput.pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN); - ASSERT(taosHashGetSize(rebOutput.pSub->consumerHash) == 0); taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); @@ -606,7 +610,6 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { // if add more consumer to balanced subscribe, // possibly no vg is changed - /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { mError("mq rebalance persist rebalance output error, possibly vnode splitted or dropped"); @@ -693,6 +696,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_OUT_OF_MEMORY; void *buf = NULL; int32_t tlen = tEncodeSubscribeObj(NULL, pSub); + if (tlen <= 0) goto SUB_ENCODE_OVER; int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 071edf992f..bf3827c090 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -384,7 +384,11 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.subType = pCreate->subType; topicObj.withMeta = pCreate->withMeta; if (topicObj.withMeta) { - ASSERT(topicObj.subType != TOPIC_SUB_TYPE__COLUMN); + if (topicObj.subType == TOPIC_SUB_TYPE__COLUMN) { + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } } if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { @@ -499,7 +503,6 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (code < 0) { sdbRelease(pSdb, pVgroup); mndTransDrop(pTrans); - ASSERT(0); return -1; } void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); @@ -723,7 +726,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { // TODO check if rebalancing if (mndDropSubByTopic(pMnode, pTrans, dropReq.name) < 0) { - /*ASSERT(0);*/ mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); mndTransDrop(pTrans); mndReleaseTopic(pMnode, pTopic); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index efce6255fb..85a92c7aef 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -884,9 +884,9 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock db = taosHashIterate(pUser->writeDbs, NULL); while (db != NULL) { cols = 0; - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)userName, false); char privilege[20] = {0}; @@ -909,9 +909,9 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock char *topic = taosHashIterate(pUser->topics, NULL); while (topic != NULL) { cols = 0; - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)userName, false); char privilege[20] = {0}; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cbee70ad03..e92daaabe8 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -54,7 +54,8 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); void vnodePreClose(SVnode *pVnode); -void vnodeSyncCheckTimeout(SVnode* pVnode); +void vnodePostClose(SVnode *pVnode); +void vnodeSyncCheckTimeout(SVnode *pVnode); void vnodeClose(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode); @@ -175,7 +176,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock* pDataBlock, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); @@ -185,7 +186,7 @@ void *tsdbGetIvtIdx(SMeta *pMeta); uint64_t getReaderMaxVersion(STsdbReader *pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, - uint64_t suid, void **pReader, const char* idstr); + uint64_t suid, void **pReader, const char *idstr); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); void *tsdbCacherowsReaderClose(void *pReader); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 5a63af41af..5a2e462c8c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -772,8 +772,8 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } -#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) -#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +// #define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +// #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { if (pIter == NULL) return NULL; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 28797c5361..93750ed585 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -98,6 +98,7 @@ bool vnodeShouldRollback(SVnode* pVnode); int32_t vnodeSyncOpen(SVnode* pVnode, char* path); int32_t vnodeSyncStart(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode); +void vnodeSyncPostClose(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode); void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code); bool vnodeIsLeader(SVnode* pVnode); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index f476f58b56..3ad01e2370 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -284,8 +284,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.pRef = walOpenRef(pTq->pVnode->pWal); if (handle.pRef == NULL) { - ASSERT(0); - return -1; + continue; } walRefVer(handle.pRef, handle.snapshotVer); diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index dd56c165fd..9e04b70f41 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -46,21 +46,37 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { } int32_t size = htonl(head.size); void* memBuf = taosMemoryCalloc(1, size); + if (memBuf == NULL) { + return -1; + } if ((code = taosReadFile(pFile, memBuf, size)) != size) { - ASSERT(0); - // TODO handle error + taosMemoryFree(memBuf); + return -1; } STqOffset offset; SDecoder decoder; tDecoderInit(&decoder, memBuf, size); if (tDecodeSTqOffset(&decoder, &offset) < 0) { - ASSERT(0); + taosMemoryFree(memBuf); + tDecoderClear(&decoder); + return -1; } + tDecoderClear(&decoder); if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { - ASSERT(0); - // TODO + return -1; } + + if (offset.val.type == TMQ_OFFSET__LOG) { + STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); + if (pHandle) { + if (walRefVer(pHandle->pRef, offset.val.version) < 0) { + tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId, + pHandle->subKey, offset.val.version); + } + } + } + taosMemoryFree(memBuf); } @@ -124,7 +140,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { const char* sysErrStr = strerror(errno); tqError("vgId:%d, cannot open file %s when commit offset since %s", pStore->pTq->pVnode->config.vgId, fname, sysErrStr); - ASSERT(0); + taosMemoryFree(fname); return -1; } taosMemoryFree(fname); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 5907be576a..6d474d5aa1 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -64,7 +64,7 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl .startTs = startTs, .endTs = endTs, }; - strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN); + strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1); taosMemoryFree(name); /*tqDebug("stream delete msg, active: vgId:%d, ts:%" PRId64 " name:%s", pVnode->config.vgId, ts, name);*/ taosArrayPush(deleteReq->deleteReqs, &req); diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index d811d943ed..129a2dd8b3 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -175,6 +175,8 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { if (code) goto _err; } + int vgId = TD_VID(pWriter->pTq->pVnode); + taosMemoryFree(pWriter); *ppWriter = NULL; @@ -186,7 +188,7 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { return code; _err: - tqError("vgId:%d, tq snapshot writer close failed since %s", TD_VID(pWriter->pTq->pVnode), tstrerror(code)); + tqError("vgId:%d, tq snapshot writer close failed since %s", vgId, tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 0a7f59e429..ac27a03096 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -19,9 +19,13 @@ #define SL_MAX_LEVEL 5 // sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l) * 2 -#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4)) -#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) -#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4)) +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_GET_NODE_FORWARD(n, l) ((SMemSkipListNode *)atomic_load_ptr(&SL_NODE_FORWARD(n, l))) +#define SL_GET_NODE_BACKWARD(n, l) ((SMemSkipListNode *)atomic_load_ptr(&SL_NODE_BACKWARD(n, l))) +#define SL_SET_NODE_FORWARD(n, l, p) atomic_store_ptr(&SL_NODE_FORWARD(n, l), p) +#define SL_SET_NODE_BACKWARD(n, l, p) atomic_store_ptr(&SL_NODE_BACKWARD(n, l), p) #define SL_MOVE_BACKWARD 0x1 #define SL_MOVE_FROM_POS 0x2 @@ -246,18 +250,18 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDa if (pFrom == NULL) { // create from head or tail if (backward) { - pIter->pNode = SL_NODE_BACKWARD(pTbData->sl.pTail, 0); + pIter->pNode = SL_GET_NODE_BACKWARD(pTbData->sl.pTail, 0); } else { - pIter->pNode = SL_NODE_FORWARD(pTbData->sl.pHead, 0); + pIter->pNode = SL_GET_NODE_FORWARD(pTbData->sl.pHead, 0); } } else { // create from a key if (backward) { tbDataMovePosTo(pTbData, pos, pFrom, SL_MOVE_BACKWARD); - pIter->pNode = SL_NODE_BACKWARD(pos[0], 0); + pIter->pNode = SL_GET_NODE_BACKWARD(pos[0], 0); } else { tbDataMovePosTo(pTbData, pos, pFrom, 0); - pIter->pNode = SL_NODE_FORWARD(pos[0], 0); + pIter->pNode = SL_GET_NODE_FORWARD(pos[0], 0); } } } @@ -271,7 +275,7 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return false; } - pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0); + pIter->pNode = SL_GET_NODE_BACKWARD(pIter->pNode, 0); if (pIter->pNode == pIter->pTbData->sl.pHead) { return false; } @@ -282,7 +286,7 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return false; } - pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0); + pIter->pNode = SL_GET_NODE_FORWARD(pIter->pNode, 0); if (pIter->pNode == pIter->pTbData->sl.pTail) { return false; } @@ -335,7 +339,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; ASSERT(pPool != NULL); - pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); + pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -408,7 +412,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p if (fromPos) px = pos[pTbData->sl.level - 1]; for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { - pn = SL_NODE_BACKWARD(px, iLevel); + pn = SL_GET_NODE_BACKWARD(px, iLevel); while (pn != pTbData->sl.pHead) { tKey.version = pn->version; tKey.ts = pn->pTSRow->ts; @@ -418,7 +422,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p break; } else { px = pn; - pn = SL_NODE_BACKWARD(px, iLevel); + pn = SL_GET_NODE_BACKWARD(px, iLevel); } } @@ -438,7 +442,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p if (fromPos) px = pos[pTbData->sl.level - 1]; for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { - pn = SL_NODE_FORWARD(px, iLevel); + pn = SL_GET_NODE_FORWARD(px, iLevel); while (pn != pTbData->sl.pTail) { tKey.version = pn->version; tKey.ts = pn->pTSRow->ts; @@ -448,7 +452,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p break; } else { px = pn; - pn = SL_NODE_FORWARD(px, iLevel); + pn = SL_GET_NODE_FORWARD(px, iLevel); } } @@ -474,58 +478,53 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN int8_t level; SMemSkipListNode *pNode; SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + int64_t nSize; - // node + // create node level = tsdbMemSkipListRandLevel(&pTbData->sl); - ASSERT(pPool != NULL); - pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level)); + nSize = SL_NODE_SIZE(level); + pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize + pRow->len); if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } pNode->level = level; pNode->version = version; - pNode->pTSRow = vnodeBufPoolMalloc(pPool, pRow->len); - if (NULL == pNode->pTSRow) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } + pNode->pTSRow = (STSRow *)((char *)pNode + nSize); memcpy(pNode->pTSRow, pRow, pRow->len); - for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { - SMemSkipListNode *pn = pos[iLevel]; - SMemSkipListNode *px; - - if (forward) { - px = SL_NODE_FORWARD(pn, iLevel); - - SL_NODE_BACKWARD(pNode, iLevel) = pn; - SL_NODE_FORWARD(pNode, iLevel) = px; - } else { - px = SL_NODE_BACKWARD(pn, iLevel); - - SL_NODE_BACKWARD(pNode, iLevel) = px; - SL_NODE_FORWARD(pNode, iLevel) = pn; + // set node + if (forward) { + for (int8_t iLevel = 0; iLevel < level; iLevel++) { + SL_NODE_FORWARD(pNode, iLevel) = SL_NODE_FORWARD(pos[iLevel], iLevel); + SL_NODE_BACKWARD(pNode, iLevel) = pos[iLevel]; + } + } else { + for (int8_t iLevel = 0; iLevel < level; iLevel++) { + SL_NODE_FORWARD(pNode, iLevel) = pos[iLevel]; + SL_NODE_BACKWARD(pNode, iLevel) = SL_NODE_BACKWARD(pos[iLevel], iLevel); } } - for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { - SMemSkipListNode *pn = pos[iLevel]; - SMemSkipListNode *px; + // set forward and backward + if (forward) { + for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { + SMemSkipListNode *pNext = pos[iLevel]->forwards[iLevel]; - if (forward) { - px = SL_NODE_FORWARD(pn, iLevel); + SL_SET_NODE_FORWARD(pos[iLevel], iLevel, pNode); + SL_SET_NODE_BACKWARD(pNext, iLevel, pNode); - SL_NODE_FORWARD(pn, iLevel) = pNode; - SL_NODE_BACKWARD(px, iLevel) = pNode; - } else { - px = SL_NODE_BACKWARD(pn, iLevel); - - SL_NODE_FORWARD(px, iLevel) = pNode; - SL_NODE_BACKWARD(pn, iLevel) = pNode; + pos[iLevel] = pNode; } + } else { + for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) { + SMemSkipListNode *pPrev = pos[iLevel]->forwards[pos[iLevel]->level + iLevel]; - pos[iLevel] = pNode; + SL_SET_NODE_FORWARD(pPrev, iLevel, pNode); + SL_SET_NODE_BACKWARD(pos[iLevel], iLevel, pNode); + + pos[iLevel] = pNode; + } } pTbData->sl.size++; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 99acc24992..7b5020d395 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -76,6 +76,7 @@ static int32_t tFDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pN static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { int32_t code = 0; + int32_t lino = 0; SDFileSet dFileSet = {.fid = pReader->fid}; SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT); @@ -83,7 +84,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { pReader->fid = pSet->fid; code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pReader->pIter = NULL; tRBTreeCreate(&pReader->rbt, tFDataIterCmprFn); @@ -93,13 +94,13 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { pIter->type = SNAP_DATA_FILE_ITER; code = tsdbReadBlockIdx(pReader->pDataFReader, pIter->aBlockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (pIter->iBlockIdx = 0; pIter->iBlockIdx < taosArrayGetSize(pIter->aBlockIdx); pIter->iBlockIdx++) { pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx); code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (pIter->iBlock = 0; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) { SDataBlk dataBlk; @@ -108,7 +109,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue; code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); ASSERT(pIter->pBlockIdx->suid == pIter->bData.suid); ASSERT(pIter->pBlockIdx->uid == pIter->bData.uid); @@ -139,7 +140,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { pIter->iStt = iStt; code = tsdbReadSttBlk(pReader->pDataFReader, iStt, pIter->aSttBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (pIter->iSttBlk = 0; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) { SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); @@ -148,7 +149,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (pSttBlk->maxVer < pReader->sver) continue; code = tsdbReadSttBlockEx(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; @@ -169,13 +170,13 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { pIter++; } - tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pReader->pTsdb->pVnode), - pReader->pTsdb->path, pReader->fid); - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb snap read open file failed since %s", TD_VID(pReader->pTsdb->pVnode), - tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d, %s failed since %s", TD_VID(pReader->pTsdb->pVnode), __func__, tstrerror(code)); + } else { + tsdbInfo("vgId:%d, %s done, path:%s, fid:%d", TD_VID(pReader->pTsdb->pVnode), __func__, pReader->pTsdb->path, + pReader->fid); + } return code; } @@ -318,12 +319,14 @@ _exit: static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; - STsdb* pTsdb = pReader->pTsdb; + int32_t lino = 0; + + STsdb* pTsdb = pReader->pTsdb; while (true) { if (pReader->pDataFReader == NULL) { code = tsdbSnapReadOpenFile(pReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } if (pReader->pDataFReader == NULL) break; @@ -338,17 +341,17 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { SBlockData* pBlockData = &pReader->bData; code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(pBlockData, &id, pReader->skmTable.pTSchema, NULL, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) { code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pRowInfo->uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbSnapNextRow(pReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbSnapGetRow(pReader); if (pRowInfo == NULL) { @@ -360,21 +363,22 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } code = tsdbSnapCmprData(pReader, ppData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); break; } - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb read data for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d, %s failed since %s, path:%s", TD_VID(pTsdb->pVnode), __func__, tstrerror(code), pTsdb->path); + } return code; } static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb* pTsdb = pReader->pTsdb; SDelFile* pDelFile = pReader->fs.pDelFile; @@ -385,11 +389,11 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { // open code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // read index code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pReader->iDelIdx = 0; } @@ -405,7 +409,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->iDelIdx++; code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); int32_t size = 0; for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) { @@ -422,7 +426,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); @@ -449,11 +453,9 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { } _exit: - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed since %s, path:%s", TD_VID(pTsdb->pVnode), __func__, tstrerror(code), pTsdb->path); + } return code; } @@ -591,44 +593,39 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + int32_t lino = 0; *ppData = NULL; // read data file if (!pReader->dataDone) { code = tsdbSnapReadData(pReader, ppData); - if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); + if (*ppData) { + goto _exit; } else { - if (*ppData) { - goto _exit; - } else { - pReader->dataDone = 1; - } + pReader->dataDone = 1; } } // read del file if (!pReader->delDone) { code = tsdbSnapReadDel(pReader, ppData); - if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); + if (*ppData) { + goto _exit; } else { - if (*ppData) { - goto _exit; - } else { - pReader->delDone = 1; - } + pReader->delDone = 1; } } _exit: - tsdbDebug("vgId:%d, vnode snapshot tsdb read for %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path); - return code; - -_err: - tsdbError("vgId:%d, vnode snapshot tsdb read for %s failed since %s", TD_VID(pReader->pTsdb->pVnode), - pReader->pTsdb->path, tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed since %s, path:%s", TD_VID(pReader->pTsdb->pVnode), __func__, tstrerror(code), + pReader->pTsdb->path); + } else { + tsdbDebug("vgId:%d, %s done, path:%s", TD_VID(pReader->pTsdb->pVnode), __func__, pReader->pTsdb->path); + } return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 125db8dd92..69af536a44 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -128,14 +128,19 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { SJson *nodeInfo = tjsonCreateArray(); if (nodeInfo == NULL) return -1; if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1; + vDebug("vgId:%d, encode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.myIndex); for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) { - SJson *info = tjsonCreateObject(); + SJson *info = tjsonCreateObject(); + SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i]; if (info == NULL) return -1; - if (tjsonAddIntegerToObject(info, "nodePort", pCfg->syncCfg.nodeInfo[i].nodePort) < 0) return -1; - if (tjsonAddStringToObject(info, "nodeFqdn", pCfg->syncCfg.nodeInfo[i].nodeFqdn) < 0) return -1; - if (tjsonAddIntegerToObject(info, "nodeId", pCfg->syncCfg.nodeInfo[i].nodeId) < 0) return -1; - if (tjsonAddIntegerToObject(info, "clusterId", pCfg->syncCfg.nodeInfo[i].clusterId) < 0) return -1; + if (tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort) < 0) return -1; + if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1; + if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1; + if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1; if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; + vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, + pNode->nodeId); } return 0; @@ -248,15 +253,20 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { int arraySize = tjsonGetArraySize(nodeInfo); if (arraySize != pCfg->syncCfg.replicaNum) return -1; + vDebug("vgId:%d, decode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.myIndex); for (int i = 0; i < arraySize; ++i) { - SJson *info = tjsonGetArrayItem(nodeInfo, i); + SJson *info = tjsonGetArrayItem(nodeInfo, i); + SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; if (info == NULL) return -1; - tjsonGetNumberValue(info, "nodePort", pCfg->syncCfg.nodeInfo[i].nodePort, code); + tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code); if (code < 0) return -1; - tjsonGetStringValue(info, "nodeFqdn", pCfg->syncCfg.nodeInfo[i].nodeFqdn); + tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn); if (code < 0) return -1; - tjsonGetNumberValue(info, "nodeId", pCfg->syncCfg.nodeInfo[i].nodeId, code); - tjsonGetNumberValue(info, "clusterId", pCfg->syncCfg.nodeInfo[i].clusterId, code); + tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code); + tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); + vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, + pNode->nodeId); } tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code); diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 04a3e7d2f7..6c54c3cb5c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -105,8 +105,8 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // free info binary taosMemoryFree(data); - vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d", pInfo->config.vgId, fname, - pInfo->config.syncCfg.replicaNum); + vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d", pInfo->config.vgId, fname, + pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex); return 0; @@ -206,6 +206,8 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { } else { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } + + vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode)); if (vnodeSaveInfo(dir, &pInfo->info) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 9cfcbda890..cea85a0c11 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -48,6 +48,7 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.applied = -1; info.state.commitID = 0; + vInfo("vgId:%d, save config while create", pCfg->vgId); if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno)); return -1; @@ -79,14 +80,14 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { pCfg->replicaNum = pReq->replica; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); - vInfo("vgId:%d, save config, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex); + vInfo("vgId:%d, save config while alter, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex); for (int i = 0; i < pReq->replica; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; pNode->nodeId = pReq->replicas[i].id; pNode->nodePort = pReq->replicas[i].port; tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); - vInfo("vgId:%d, save config, replica:%d ep:%s:%u", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort); + vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } info.config.syncCfg = *pCfg; @@ -249,6 +250,8 @@ void vnodePreClose(SVnode *pVnode) { vnodeSyncPreClose(pVnode); } +void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); } + void vnodeClose(SVnode *pVnode) { if (pVnode) { vnodeSyncCommit(pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 2fc06fba86..cc22668b29 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -405,6 +405,10 @@ static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_ } else { snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path); } + + SVnode *pVnode = pWriter->pVnode; + pWriter->info.config = pVnode->config; + vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId); if (vnodeSaveInfo(dir, &pWriter->info) < 0) { code = terrno; goto _exit; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8d53579483..ac6406fc23 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -182,7 +182,6 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, pVnode->state.applied); terrno = TSDB_CODE_VND_DUP_REQUEST; - pRsp->info.handle = NULL; return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 749c81224c..65e17cfaad 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -614,6 +614,11 @@ void vnodeSyncPreClose(SVnode *pVnode) { taosThreadMutexUnlock(&pVnode->lock); } +void vnodeSyncPostClose(SVnode *pVnode) { + vInfo("vgId:%d, post close sync", pVnode->config.vgId); + syncPostStop(pVnode->sync); +} + void vnodeSyncClose(SVnode *pVnode) { vInfo("vgId:%d, close sync", pVnode->config.vgId); syncStop(pVnode->sync); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a6ce613882..3e8300e05d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -776,6 +776,7 @@ void ctgFreeHandleImpl(SCatalog* pCtg); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); +int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index acd0fb9d8c..c7af0411be 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -551,6 +551,37 @@ _return: CTG_RET(code); } +int32_t ctgGetTbsHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctId, const char* pDb, const char* pTbs[], int32_t tbNum, int32_t* vgId) { + if (IS_SYS_DBNAME(pDb)) { + ctgError("no valid vgInfo for db, dbname:%s", pDb); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + SCtgDBCache* dbCache = NULL; + int32_t code = 0; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb); + + SDBVgInfo* vgInfo = NULL; + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo, NULL)); + + CTG_ERR_JRET(ctgGetVgIdsFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, dbFName, pTbs, tbNum, vgId)); + +_return: + + if (dbCache) { + ctgRUnlockVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } + + if (vgInfo) { + freeVgInfo(vgInfo); + } + + CTG_RET(code); +} + + int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) { int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; @@ -1141,6 +1172,13 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, NULL)); } +int32_t catalogGetTablesHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctId, const char* pDb, const char* pTableName[], + int32_t tableNum, int32_t *vgId) { + CTG_API_ENTER(); + + CTG_API_LEAVE(ctgGetTbsHashVgId(pCtg, pConn, acctId, pDb, pTableName, tableNum, vgId)); +} + int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 802ecde63e..3dd40a4139 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -986,6 +986,43 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* CTG_RET(code); } +int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId) { + int32_t code = 0; + CTG_ERR_RET(ctgMakeVgArray(dbInfo)); + + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); + + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + SVgroupInfo* vgInfo = NULL; + char tbFullName[TSDB_TABLE_FNAME_LEN]; + snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); + int32_t offset = strlen(tbFullName); + + for (int32_t i = 0; i < tbNum; ++i) { + snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]); + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, + dbInfo->hashPrefix, dbInfo->hashSuffix); + + vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); + if (NULL == vgInfo) { + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, + (int32_t)taosArrayGetSize(dbInfo->vgArray)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + vgId[i] = vgInfo->vgId; + + ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId); + } + + CTG_RET(code); +} + + int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { if (*(uint64_t*)key1 < ((SSTableVersion*)key2)->suid) { return -1; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 4229e8808d..e0d2276e6f 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -115,6 +115,10 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); + if (NULL == bufPage) { + return NULL; + } + if (forUpdate) { setBufPageDirty(bufPage, true); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fc3cfbd0f6..a5468008aa 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1726,8 +1726,10 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI return w; } - w = getResultRowByPos(pBuf, &pResultRowInfo->cur, false)->win; - + SResultRow* pRow = getResultRowByPos(pBuf, &pResultRowInfo->cur, false); + if (pRow) { + w = pRow->win; + } // in case of typical time window, we can calculate time window directly. if (w.skey > ts || w.ekey < ts) { w = doCalculateTimeWindow(ts, pInterval); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e5ff104d5c..814ead57f0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -704,6 +704,15 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { return TSDB_CODE_SUCCESS; } +bool qTaskIsExecuting(qTaskInfo_t qinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; + if (NULL == pTaskInfo) { + return false; + } + + return 0 != atomic_load_64(&pTaskInfo->owner); +} + static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { STaskCostInfo* pSummary = &pTaskInfo->cost; int64_t idleTime = pSummary->start - pSummary->created; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 57e6a63137..448e2b0a91 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -150,6 +150,11 @@ SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, i pData->num = sizeof(SFilePage); } else { pData = getBufPage(pResultBuf, *currentPageId); + if (pData == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return NULL; + } + pageId = *currentPageId; if (pData->num + interBufSize > getBufPageSize(pResultBuf)) { @@ -200,6 +205,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (isIntervalQuery) { if (p1 != NULL) { // the *p1 may be NULL in case of sliding+offset exists. pResult = getResultRowByPos(pResultBuf, p1, true); + if (NULL == pResult) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); } } else { @@ -208,6 +217,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (p1 != NULL) { // todo pResult = getResultRowByPos(pResultBuf, p1, true); + if (NULL == pResult) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); } } @@ -216,6 +229,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId))) { SResultRowPosition pos = pResultRowInfo->cur; SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); + if (pPage == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } releaseBufPage(pResultBuf, pPage); } @@ -223,6 +240,9 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (pResult == NULL) { ASSERT(pSup->resultRowSize > 0); pResult = getNewResultRow(pResultBuf, &pSup->currentPageId, pSup->resultRowSize); + if (pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, terrno); + } // add a new result set for a new group SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; @@ -260,6 +280,11 @@ static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pRes } else { SPageInfo* pi = getLastPageInfo(list); pData = getBufPage(pResultBuf, getPageId(pi)); + if (pData == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return terrno; + } + pageId = getPageId(pi); if (pData->num + size > getBufPageSize(pResultBuf)) { @@ -846,6 +871,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD return; } + int8_t* pIndicator = (int8_t*)p->pData; int32_t totalRows = pBlock->info.rows; if (status == FILTER_RESULT_ALL_QUALIFIED) { @@ -853,42 +879,135 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD } else if (status == FILTER_RESULT_NONE_QUALIFIED) { pBlock->info.rows = 0; } else { - SSDataBlock* px = createOneDataBlock(pBlock, true); + int32_t bmLen = BitmapLen(totalRows); + char* pBitmap = NULL; + int32_t maxRows = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); // it is a reserved column for scalar function, and no data in this column yet. - if (pDst->pData == NULL || pSrc->pData == NULL) { + if (pDst->pData == NULL) { continue; } - colInfoDataCleanup(pDst, pBlock->info.rows); - int32_t numOfRows = 0; - for (int32_t j = 0; j < totalRows; ++j) { - if (((int8_t*)p->pData)[j] == 0) { - continue; + + if (IS_VAR_DATA_TYPE(pDst->info.type)) { + int32_t j = 0; + pDst->varmeta.length = 0; + + while(j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + + if (colDataIsNull_var(pDst, j)) { + colDataSetNull_var(pDst, numOfRows); + } else { + char* p1 = colDataGetVarData(pDst, j); + colDataAppend(pDst, numOfRows, p1, false); + } + numOfRows += 1; + j += 1; } - if (colDataIsNull_s(pSrc, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - colDataAppend(pDst, numOfRows, colDataGetData(pSrc, j), false); + if (maxRows < numOfRows) { + maxRows = numOfRows; + } + } else { + if (pBitmap == NULL) { + pBitmap = taosMemoryCalloc(1, bmLen); + } + + memcpy(pBitmap, pDst->nullbitmap, bmLen); + memset(pDst->nullbitmap, 0, bmLen); + + int32_t j = 0; + + switch (pDst->info.type) { + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int32_t*)pDst->pData)[numOfRows] = ((int32_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int16_t*)pDst->pData)[numOfRows] = ((int16_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; } - numOfRows += 1; } - // todo this value can be assigned directly - if (pBlock->info.rows == totalRows) { - pBlock->info.rows = numOfRows; - } else { - ASSERT(pBlock->info.rows == numOfRows); + if (maxRows < numOfRows) { + maxRows = numOfRows; } } - blockDataDestroy(px); // fix memory leak + pBlock->info.rows = maxRows; + if (pBitmap != NULL) { + taosMemoryFree(pBitmap); + } } } @@ -912,7 +1031,7 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin if (pResultRow->pageId == -1) { int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize); if (ret != TSDB_CODE_SUCCESS) { - return; + T_LONG_JMP(pTaskInfo->env, terrno); } } @@ -993,6 +1112,11 @@ static void doCopyResultToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SR int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } + SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset); SqlFunctionCtx* pCtx = pSup->pCtx; @@ -1036,6 +1160,10 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 21ec5afdd6..5676e19cdf 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -492,13 +492,17 @@ _error: static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { SPartitionOperatorInfo* pInfo = pOperator->info; - + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); SDataGroupInfo* pGroupInfo = NULL; void* pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); + if (pPage == NULL) { + T_LONG_JMP(pTaskInfo->env, terrno); + } pGroupInfo->numOfRows += 1; @@ -595,6 +599,10 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf } else { int32_t* curId = taosArrayGetLast(p->pPageList); pPage = getBufPage(pInfo->pBuf, *curId); + if (pPage == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return pPage; + } int32_t* rows = (int32_t*)pPage; if (*rows >= pInfo->rowCapacity) { @@ -674,7 +682,8 @@ static int compareDataGroupInfo(const void* group1, const void* group2) { static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; - + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SDataGroupInfo* pGroupInfo = (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { @@ -692,7 +701,11 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex); void* page = getBufPage(pInfo->pBuf, *pageId); - + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } + blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity); blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0e22195afa..1d7f27d0cf 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -170,6 +170,10 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro } *pPage = getBufPage(pTableScanInfo->base.pdInfo.pAggSup->pResultBuf, p1->pageId); + if (NULL == *pPage) { + return NULL; + } + return (SResultRow*)((char*)(*pPage) + p1->offset); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 58fadc60b0..d78e9c4edf 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -636,6 +636,10 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false); + if (NULL == pr) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId); if (pr->closed) { @@ -1315,6 +1319,10 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false); + if (NULL == pResult) { + return; + } + SqlFunctionCtx* pCtx = pSup->pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset); @@ -1328,6 +1336,9 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, } } SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId); + if (NULL == bufPage) { + return; + } setBufPageDirty(bufPage, true); releaseBufPage(pResultBuf, bufPage); } @@ -4114,6 +4125,9 @@ void destroyMAIOperatorInfo(void* param) { static SResultRow* doSetSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, SAggSupporter* pSup) { SResultRow* pResult = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize); + if (NULL == pResult) { + return pResult; + } pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; return pResult; } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index fa0cdb3943..06ef36664a 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -270,6 +270,10 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); void* pPage = getBufPage(pHandle->pBuf, *pPgId); + if (NULL == pPage) { + return terrno; + } + code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { return code; @@ -337,6 +341,11 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); void* pPage = getBufPage(pHandle->pBuf, *pPgId); + if (pPage == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return terrno; + } + int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index cbda8dc472..dc884a0581 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -44,9 +44,10 @@ typedef struct SMinmaxResInfo { bool nullTupleSaved; int16_t type; } SMinmaxResInfo; -int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc); -STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock); +int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems); + +int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 07e480ee1d..faf7a29dd0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1009,7 +1009,10 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err intervals[0] = -INFINITY; intervals[numOfBins - 1] = INFINITY; // in case of desc bin orders, -inf/inf should be swapped - ASSERT(numOfBins >= 4); + if (numOfBins < 4) { + return false; + } + if (intervals[1] > intervals[numOfBins - 2]) { TSWAP(intervals[0], intervals[numOfBins - 1]); } @@ -1354,7 +1357,7 @@ static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } else if (IS_FLOAT_TYPE(colType)) { resType = TSDB_DATA_TYPE_DOUBLE; } else { - ASSERT(0); + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 21aff305c6..8fde27e046 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -506,7 +506,6 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { SColumnInfoData* pInputCol = pInput->pData[0]; if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; - ASSERT(numOfElem >= 0); } else { if (pInputCol->hasNull) { for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { @@ -596,7 +595,6 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet) { numOfElem = pInput->numOfRows - pAgg->numOfNull; - ASSERT(numOfElem >= 0); if (IS_SIGNED_NUMERIC_TYPE(type)) { pSumRes->isum += pAgg->sum; @@ -661,7 +659,6 @@ int32_t sumInvertFunction(SqlFunctionCtx* pCtx) { if (pInput->colDataSMAIsSet) { numOfElem = pInput->numOfRows - pAgg->numOfNull; - ASSERT(numOfElem >= 0); if (IS_SIGNED_NUMERIC_TYPE(type)) { pSumRes->isum -= pAgg->sum; @@ -757,24 +754,33 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { } int32_t minFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElems = doMinMaxHelper(pCtx, 1); + int32_t numOfElems = 0; + int32_t code = doMinMaxHelper(pCtx, 1, &numOfElems); + if (code != TSDB_CODE_SUCCESS) { + return code; + } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); return TSDB_CODE_SUCCESS; } int32_t maxFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElems = doMinMaxHelper(pCtx, 0); + int32_t numOfElems = 0; + int32_t code = doMinMaxHelper(pCtx, 0, &numOfElems); + if (code != TSDB_CODE_SUCCESS) { + return code; + } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); return TSDB_CODE_SUCCESS; } -static void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex); -static void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, +static int32_t setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex); +static int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex); int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); + int32_t code = TSDB_CODE_SUCCESS; + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); SMinmaxResInfo* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; @@ -784,24 +790,24 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pEntryInfo->isNullRes = (pEntryInfo->numOfRes == 0) ? 1 : 0; if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = *(float*)&pRes->v; + float v = GET_FLOAT_VAL(&pRes->v); colDataAppend(pCol, currentRow, (const char*)&v, pEntryInfo->isNullRes); } else { colDataAppend(pCol, currentRow, (const char*)&pRes->v, pEntryInfo->isNullRes); } if (pEntryInfo->numOfRes > 0) { - setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); } else { - setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); } - return pEntryInfo->numOfRes; + return code; } -void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex) { +int32_t setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex) { if (pCtx->subsidiaries.num <= 0) { - return; + return TSDB_CODE_SUCCESS; } for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { @@ -811,17 +817,23 @@ void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); colDataAppendNULL(pDstCol, rowIndex); } + + return TSDB_CODE_SUCCESS; } -void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { +int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { if (pCtx->subsidiaries.num <= 0) { - return; + return TSDB_CODE_SUCCESS; } if ((pCtx->saveHandle.pBuf != NULL && pTuplePos->pageId != -1) || (pCtx->saveHandle.pState && pTuplePos->streamTupleKey.ts > 0)) { int32_t numOfCols = pCtx->subsidiaries.num; const char* p = loadTupleData(pCtx, pTuplePos); + if (p == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } bool* nullList = (bool*)p; char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); @@ -832,7 +844,6 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); - ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes); if (nullList[j]) { colDataAppendNULL(pDstCol, rowIndex); } else { @@ -845,6 +856,8 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple tdbFree((void*)p); } } + + return TSDB_CODE_SUCCESS; } void releaseSource(STuplePos* pPos) { @@ -876,7 +889,6 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pCtx->pDstBlock->pDataBlock, dstSlotId); - ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes); if (colDataIsNull_s(pSrcCol, rowIndex) == true) { colDataAppendNULL(pDstCol, pos); @@ -1145,7 +1157,9 @@ static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) { int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -1570,7 +1584,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { // all data are null, set it completed if (pInfo->numOfElems == 0) { pResInfo->complete = true; - return 0; + return TSDB_CODE_SUCCESS; } else { pInfo->pMemBucket = tMemBucketCreate(pCol->info.bytes, type, pInfo->minval, pInfo->maxval); } @@ -1633,7 +1647,11 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pCol, i); numOfElems += 1; - tMemBucketPut(pInfo->pMemBucket, data, 1); + int32_t code = tMemBucketPut(pInfo->pMemBucket, data, 1); + if (code != TSDB_CODE_SUCCESS) { + tMemBucketDestroy(pInfo->pMemBucket); + return code; + } } SET_VAL(pResInfo, numOfElems, 1); @@ -1656,6 +1674,11 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } tMemBucketDestroy(pMemBucket); + + if (ppInfo->result < 0) { + return TSDB_CODE_NO_AVAIL_DISK; + } + return functionFinalize(pCtx, pBlock); } @@ -1771,7 +1794,10 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { double v = 0; GET_TYPED_DATA(v, double, type, data); - tHistogramAdd(&pInfo->pHisto, v); + int32_t code = tHistogramAdd(&pInfo->pHisto, v); + if (code != 0) { + return TSDB_CODE_FAILED; + } } qDebug("%s after add %d elements into histogram, total:%" PRId64 ", numOfEntry:%d, pHisto:%p, elems: %p", @@ -1841,7 +1867,9 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2009,25 +2037,26 @@ static void prepareBuf(SqlFunctionCtx* pCtx) { pCtx->subsidiaries.rowLen = rowLen + pCtx->subsidiaries.num * sizeof(bool); pCtx->subsidiaries.buf = taosMemoryMalloc(pCtx->subsidiaries.rowLen); } - - ASSERT(pCtx->subsidiaries.buf != NULL); - ASSERT(pCtx->subsidiaries.rowLen > 0); } -static void firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, +static int32_t firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, SFirstLastRes* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + if (pCtx->subsidiaries.num <= 0) { - return; + return TSDB_CODE_SUCCESS; } if (!pInfo->hasResult) { - pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock); + code = saveTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); } else { - updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + code = updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); } + + return code; } -static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t currentTs, int32_t type, char* pData) { +static int32_t doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t currentTs, int32_t type, char* pData) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2037,9 +2066,13 @@ static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t cur memcpy(pInfo->buf, pData, pInfo->bytes); pInfo->ts = currentTs; - firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->hasResult = true; + return TSDB_CODE_SUCCESS; } // This ordinary first function does not care if current scan is ascending order or descending order scan @@ -2060,10 +2093,12 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } // All null data column, return directly. - if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { - ASSERT(pInputCol->hasNull == true); + if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } return TSDB_CODE_SUCCESS; } @@ -2136,7 +2171,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = pts[i]; if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { - doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); + int32_t code = doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } @@ -2144,7 +2182,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (numOfElems == 0) { // save selectivity value for column consisted of all null values - firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -2168,10 +2209,12 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } // All null data column, return directly. - if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { - ASSERT(pInputCol->hasNull == true); + if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) && pInputCol->hasNull == true) { // save selectivity value for column consisted of all null values - firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } return TSDB_CODE_SUCCESS; } @@ -2261,7 +2304,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { char* data = colDataGetData(pInputCol, chosen); - doSaveCurrentVal(pCtx, i, cts, type, data); + int32_t code = doSaveCurrentVal(pCtx, i, cts, type, data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } @@ -2269,7 +2315,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { char* data = colDataGetData(pInputCol, i); - doSaveCurrentVal(pCtx, i, pts[i], type, data); + int32_t code = doSaveCurrentVal(pCtx, i, pts[i], type, data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } @@ -2283,7 +2332,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { char* data = colDataGetData(pInputCol, i); - doSaveCurrentVal(pCtx, i, pts[i], type, data); + int32_t code = doSaveCurrentVal(pCtx, i, pts[i], type, data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } @@ -2294,7 +2346,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { // save selectivity value for column consisted of all null values if (numOfElems == 0) { - firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } // SET_VAL(pResInfo, numOfElems, 1); @@ -2322,18 +2377,25 @@ static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* p return TSDB_CODE_SUCCESS; } -static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, +static int32_t firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, int32_t rowIndex) { if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pInput, pOutput, isFirst)) { - firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pOutput->hasResult = true; } + + return TSDB_CODE_SUCCESS; } static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -2343,7 +2405,10 @@ static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuer for (int32_t i = start; i < start + pInput->numOfRows; ++i) { char* data = colDataGetData(pCol, i); SFirstLastRes* pInputInfo = (SFirstLastRes*)varDataVal(data); - firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery, i); + int32_t code = firstLastTransferInfo(pCtx, pInputInfo, pInfo, isFirstQuery, i); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (!numOfElems) { numOfElems = pInputInfo->hasResult ? 1 : 0; } @@ -2358,6 +2423,7 @@ int32_t firstFunctionMerge(SqlFunctionCtx* pCtx) { return firstLastFunctionMerge int32_t lastFunctionMerge(SqlFunctionCtx* pCtx) { return firstLastFunctionMergeImpl(pCtx, false); } int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -2368,12 +2434,14 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppend(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); // handle selectivity - setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); + code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); - return pResInfo->numOfRes; + return code; } int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); @@ -2389,10 +2457,10 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); colDataAppend(pCol, pBlock->info.rows, res, false); - setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); + code = setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); taosMemoryFree(res); - return 1; + return code; } int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { @@ -2412,7 +2480,7 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { return TSDB_CODE_SUCCESS; } -static void doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex, int64_t cts, SFirstLastRes* pInfo) { +static int32_t doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex, int64_t cts, SFirstLastRes* pInfo) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; @@ -2429,9 +2497,14 @@ static void doSaveLastrow(SqlFunctionCtx* pCtx, char* pData, int32_t rowIndex, i } pInfo->ts = cts; - firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + int32_t code = firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->hasResult = true; + + return TSDB_CODE_SUCCESS; } int32_t lastRowFunction(SqlFunctionCtx* pCtx) { @@ -2493,7 +2566,10 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveLastrow(pCtx, data, i, cts, pInfo); + int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } @@ -2528,7 +2604,7 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { return true; } -static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int64_t ts) { +static int32_t doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int64_t ts) { switch (type) { case TSDB_DATA_TYPE_BOOL: pDiffInfo->prev.i64 = *(bool*)pv ? 1 : 0; @@ -2553,12 +2629,14 @@ static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv, int pDiffInfo->prev.d64 = *(double*)pv; break; default: - ASSERT(0); + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } pDiffInfo->prevTs = ts; + + return TSDB_CODE_SUCCESS; } -static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos, +static int32_t doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SColumnInfoData* pOutput, int32_t pos, int32_t order, int64_t ts) { int32_t factor = (order == TSDB_ORDER_ASC) ? 1 : -1; pDiffInfo->prevTs = ts; @@ -2633,8 +2711,10 @@ static void doHandleDiff(SDiffInfo* pDiffInfo, int32_t type, const char* pv, SCo break; } default: - ASSERT(0); + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } + + return TSDB_CODE_SUCCESS; } int32_t diffFunction(SqlFunctionCtx* pCtx) { @@ -2671,7 +2751,10 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { if (tsList[i] == pDiffInfo->prevTs) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); + int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // handle selectivity if (pCtx->subsidiaries.num > 0) { appendSelectivityValue(pCtx, i, pos); @@ -2679,7 +2762,10 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { numOfElems++; } else { - doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); + int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pDiffInfo->hasPrev = true; @@ -2704,7 +2790,10 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { if (tsList[i] == pDiffInfo->prevTs) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } - doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); + int32_t code = doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } // handle selectivity if (pCtx->subsidiaries.num > 0) { appendSelectivityValue(pCtx, i, pos); @@ -2712,7 +2801,10 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { numOfElems++; } else { - doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); + int32_t code = doSetPrevVal(pDiffInfo, pInputCol->info.type, pv, tsList[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pDiffInfo->hasPrev = true; @@ -2754,8 +2846,8 @@ static STopBotRes* getTopBotOutputInfo(SqlFunctionCtx* pCtx) { return pRes; } -static void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, - uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery); +static int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, + uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery); static void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, bool isTopQuery); @@ -2777,11 +2869,17 @@ int32_t topFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pCol, i); - doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, true); + int32_t code = doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, true); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { - pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pRes->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pRes->nullTupleSaved = true; } return TSDB_CODE_SUCCESS; @@ -2805,11 +2903,17 @@ int32_t bottomFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pCol, i); - doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, false); + int32_t code = doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, false); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { - pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pRes->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pRes->nullTupleSaved = true; } @@ -2849,7 +2953,7 @@ static int32_t topBotResComparFn(const void* p1, const void* p2, const void* par return (val1->v.d > val2->v.d) ? 1 : -1; } -void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, +int32_t doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo, bool isTopQuery) { STopBotRes* pRes = getTopBotOutputInfo(pCtx); @@ -2857,7 +2961,6 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); STopBotResItem* pItems = pRes->pItems; - assert(pItems != NULL); // not full yet if (pEntryInfo->numOfRes < pRes->maxSize) { @@ -2867,7 +2970,10 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple if (pCtx->subsidiaries.num > 0) { - pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock); + int32_t code = saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } #ifdef BUF_PAGE_DEBUG qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId, @@ -2893,7 +2999,10 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple by over writing the old data if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + int32_t code = updateTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } #ifdef BUF_PAGE_DEBUG qDebug("page_copyTuple pageId:%d, offset:%d", pItem->tuplePos.pageId, pItem->tuplePos.offset); @@ -2902,6 +3011,8 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData topBotResComparFn, NULL, !isTopQuery); } } + + return TSDB_CODE_SUCCESS; } /* @@ -2941,20 +3052,33 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid return buf; } -static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STupleKey key) { +static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STupleKey key, + STuplePos* pPos) { STuplePos p = {0}; if (pHandle->pBuf != NULL) { SFilePage* pPage = NULL; if (pHandle->currentPage == -1) { pPage = getNewBufPage(pHandle->pBuf, &pHandle->currentPage); + if (pPage == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } pPage->num = sizeof(SFilePage); } else { pPage = getBufPage(pHandle->pBuf, pHandle->currentPage); + if (pPage == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } if (pPage->num + length > getBufPageSize(pHandle->pBuf)) { // current page is all used, let's prepare a new buffer page releaseBufPage(pHandle->pBuf, pPage); pPage = getNewBufPage(pHandle->pBuf, &pHandle->currentPage); + if (pPage == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } pPage->num = sizeof(SFilePage); } } @@ -2967,36 +3091,41 @@ static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf releaseBufPage(pHandle->pBuf, pPage); } else { // other tuple save policy - if (streamStateFuncPut(pHandle->pState, &key, pBuf, length) < 0) { - ASSERT(0); + if (streamStateFuncPut(pHandle->pState, &key, pBuf, length) >= 0) { + p.streamTupleKey = key; } - p.streamTupleKey = key; } - return p; + *pPos = p; + return TSDB_CODE_SUCCESS; } -STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock) { +int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { prepareBuf(pCtx); STupleKey key; if (pCtx->saveHandle.pBuf == NULL) { SColumnInfoData* pColInfo = taosArrayGet(pSrcBlock->pDataBlock, 0); - ASSERT(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); - int64_t skey = *(int64_t*)colDataGetData(pColInfo, rowIndex); + if (pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t skey = *(int64_t*)colDataGetData(pColInfo, rowIndex); - key.groupId = pSrcBlock->info.id.groupId; - key.ts = skey; - key.exprIdx = pCtx->exprIdx; + key.groupId = pSrcBlock->info.id.groupId; + key.ts = skey; + key.exprIdx = pCtx->exprIdx; + } } char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, key); + return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, key, pPos); } static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); + if (pPage == NULL) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return terrno; + } memcpy(pPage->data + pPos->offset, pBuf, length); setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); @@ -3011,13 +3140,15 @@ int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc prepareBuf(pCtx); char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos); - return TSDB_CODE_SUCCESS; + return doUpdateTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pPos); } static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); + if (pPage == NULL) { + return NULL; + } char* p = pPage->data + pPos->offset; releaseBufPage(pHandle->pBuf, pPage); return p; @@ -3034,6 +3165,8 @@ const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos) { } int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); STopBotRes* pRes = getTopBotOutputInfo(pCtx); @@ -3046,8 +3179,8 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pEntryInfo->numOfRes <= 0) { colDataAppendNULL(pCol, currentRow); - setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); - return pEntryInfo->numOfRes; + code = setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); + return code; } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; @@ -3056,18 +3189,17 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { qDebug("page_finalize i:%d,item:%p,pageId:%d, offset:%d\n", i, pItem, pItem->tuplePos.pageId, pItem->tuplePos.offset); #endif - setSelectivityValue(pCtx, pBlock, &pRes->pItems[i].tuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pRes->pItems[i].tuplePos, currentRow); currentRow += 1; } - return pEntryInfo->numOfRes; + return code; } void addResult(SqlFunctionCtx* pCtx, STopBotResItem* pSourceItem, int16_t type, bool isTopQuery) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); STopBotRes* pRes = getTopBotOutputInfo(pCtx); STopBotResItem* pItems = pRes->pItems; - assert(pItems != NULL); // not full yet if (pEntryInfo->numOfRes < pRes->maxSize) { @@ -3225,7 +3357,9 @@ static void spreadTransferInfo(SSpreadInfo* pInput, SSpreadInfo* pOutput) { int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -3404,7 +3538,9 @@ static void elapsedTransferInfo(SElapsedInfo* pInput, SElapsedInfo* pOutput) { int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -3574,7 +3710,9 @@ static bool getHistogramBinDesc(SHistoFuncInfo* pInfo, char* binDescStr, int8_t intervals[0] = -INFINITY; intervals[numOfBins - 1] = INFINITY; // in case of desc bin orders, -inf/inf should be swapped - ASSERT(numOfBins >= 4); + if (numOfBins < 4) { + return false; + } if (intervals[1] > intervals[numOfBins - 2]) { TSWAP(intervals[0], intervals[numOfBins - 1]); } @@ -3717,7 +3855,9 @@ static void histogramTransferInfo(SHistoFuncInfo* pInput, SHistoFuncInfo* pOutpu int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -4099,7 +4239,7 @@ static bool checkStateOp(int8_t op, SColumnInfoData* pCol, int32_t index, SVaria break; } default: { - ASSERT(0); + return false; } } return false; @@ -4431,12 +4571,15 @@ static void sampleAssignResult(SSampleInfo* pInfo, char* data, int32_t index) { assignVal(pInfo->data + index * pInfo->colBytes, data, pInfo->colBytes, pInfo->colType); } -static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* data, int32_t index) { +static int32_t doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* data, int32_t index) { pInfo->totalPoints++; if (pInfo->numSampled < pInfo->samples) { sampleAssignResult(pInfo, data, pInfo->numSampled); if (pCtx->subsidiaries.num > 0) { - pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[pInfo->numSampled]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pInfo->numSampled++; } else { @@ -4444,10 +4587,15 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da if (j < pInfo->samples) { sampleAssignResult(pInfo, data, j); if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); + int32_t code = updateTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } + + return TSDB_CODE_SUCCESS; } int32_t sampleFunction(SqlFunctionCtx* pCtx) { @@ -4463,11 +4611,17 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { } char* data = colDataGetData(pInputCol, i); - doReservoirSample(pCtx, pInfo, data, i); + int32_t code = doReservoirSample(pCtx, pInfo, data, i); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (pInfo->numSampled == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { - pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pInfo->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->nullTupleSaved = true; } @@ -4476,6 +4630,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { } int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); SSampleInfo* pInfo = getSampleOutputInfo(pCtx); @@ -4487,15 +4642,15 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; if (pInfo->numSampled == 0) { colDataAppendNULL(pCol, currentRow); - setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); - return pInfo->numSampled; + code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); + return code; } for (int32_t i = 0; i < pInfo->numSampled; ++i) { colDataAppend(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); - setSelectivityValue(pCtx, pBlock, &pInfo->tuplePos[i], currentRow + i); + code = setSelectivityValue(pCtx, pBlock, &pInfo->tuplePos[i], currentRow + i); } - return pInfo->numSampled; + return code; } bool getTailFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -4761,7 +4916,7 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { return true; } -static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { +static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { @@ -4771,7 +4926,10 @@ static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, pItem->count += 1; if (pCtx->subsidiaries.num > 0) { - pItem->tuplePos = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pItem->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*)); @@ -4779,9 +4937,14 @@ static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, } else { (*pHashItem)->count += 1; if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &((*pHashItem)->tuplePos)); + int32_t code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &((*pHashItem)->tuplePos)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } + + return TSDB_CODE_SUCCESS; } int32_t modeFunction(SqlFunctionCtx* pCtx) { @@ -4802,7 +4965,10 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pInputCol, i); - doModeAdd(pInfo, i, pCtx, data); + int32_t code = doModeAdd(pInfo, i, pCtx, data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { taosHashCleanup(pInfo->pHash); @@ -4811,7 +4977,10 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { - pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pInfo->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pInfo->nullTupleSaved = true; } @@ -4821,6 +4990,7 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; @@ -4840,15 +5010,15 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (maxCount != 0) { SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); colDataAppend(pCol, currentRow, pResItem->data, false); - setSelectivityValue(pCtx, pBlock, &pResItem->tuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pResItem->tuplePos, currentRow); } else { colDataAppendNULL(pCol, currentRow); - setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); + code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); } taosHashCleanup(pInfo->pHash); - return pResInfo->numOfRes; + return code; } bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -4900,11 +5070,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } int32_t i = pInput->startRowIndex; - if (pCtx->start.key != INT64_MIN) { - // ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - // (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); - - ASSERT(last->key == INT64_MIN); + if (pCtx->start.key != INT64_MIN && last->key == INT64_MIN) { for (; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; @@ -5124,7 +5290,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } default: - ASSERT(0); + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } // the last interpolated time window value @@ -5151,7 +5317,6 @@ _twa_over: * is required, we simply copy to the resut ot interResBuffer. */ // void twa_function_copy(SQLFunctionCtx *pCtx) { -// assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); // SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // // memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); @@ -5697,7 +5862,10 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveLastrow(pCtx, data, i, cts, pInfo); + int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pResInfo->numOfRes = 1; } } diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 8e010181d1..3a70a65ec4 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -366,7 +366,6 @@ bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { static int32_t calculateAvgBySMAInfo(SAvgRes* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg) { int32_t numOfElem = numOfRows - pAgg->numOfNull; - ASSERT(numOfElem >= 0); pRes->count += numOfElem; if (IS_SIGNED_NUMERIC_TYPE(type)) { @@ -672,7 +671,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { break; } default: - ASSERT(0); + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; } } else { numOfElem = doAddNumericVector(pCol, type, pInput, pAvgRes); @@ -706,7 +705,9 @@ static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) { int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_FUNC_FUNTION_PARA_TYPE; + } SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); diff --git a/source/libs/function/src/detail/tminmax.c b/source/libs/function/src/detail/tminmax.c index 05cebfeca7..847c738655 100644 --- a/source/libs/function/src/detail/tminmax.c +++ b/source/libs/function/src/detail/tminmax.c @@ -700,7 +700,7 @@ static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunct } } -int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { +int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) { int32_t numOfElems = 0; SInputColumnInfoData* pInput = &pCtx->input; @@ -721,7 +721,6 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { // data in current data block are qualified to the query if (pInput->colDataSMAIsSet) { numOfElems = pInput->numOfRows - pAgg->numOfNull; - ASSERT(pInput->numOfRows == pInput->totalRows && numOfElems >= 0); if (numOfElems == 0) { goto _over; @@ -737,11 +736,19 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } if (!pBuf->assign) { - pBuf->v = *(int64_t*)tval; + if (type == TSDB_DATA_TYPE_FLOAT) { + GET_FLOAT_VAL(&pBuf->v) = GET_DOUBLE_VAL(tval); + } else { + pBuf->v = GET_INT64_VAL(tval); + } + if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } else { @@ -751,11 +758,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { int64_t val = GET_INT64_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(int64_t*)&pBuf->v = val; + GET_INT64_VAL(&pBuf->v) = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } @@ -765,11 +775,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { uint64_t val = GET_UINT64_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(uint64_t*)&pBuf->v = val; + GET_UINT64_VAL(&pBuf->v) = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } @@ -779,11 +792,14 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { double val = GET_DOUBLE_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(double*)&pBuf->v = val; + GET_DOUBLE_VAL(&pBuf->v) = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } @@ -793,20 +809,23 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { float val = GET_DOUBLE_VAL(tval); if ((prev < val) ^ isMinFunc) { - *(float*)&pBuf->v = val; + GET_FLOAT_VAL(&pBuf->v) = val; } if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); if (index >= 0) { - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } } } pBuf->assign = true; - return numOfElems; + return TSDB_CODE_SUCCESS; } int32_t start = pInput->startRowIndex; @@ -820,14 +839,16 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { memcpy(&pBuf->v, pCol->pData + (pCol->info.bytes * i), pCol->info.bytes); if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } pBuf->assign = true; numOfElems = 1; } if (i >= end) { - ASSERT(numOfElems == 0); goto _over; } @@ -884,9 +905,13 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { _over: if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) { - pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); + int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pBuf->nullTuplePos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pBuf->nullTupleSaved = true; } - return numOfElems; + *nElems = numOfElems; + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c index 3afa0e5cdd..edfd866010 100644 --- a/source/libs/function/src/tfunctionInt.c +++ b/source/libs/function/src/tfunctionInt.c @@ -40,8 +40,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock } } - assert(maxRows >= 0); - blockDataEnsureCapacity(pResBlock, maxRows); for (int32_t i = 0; i < num; ++i) { SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); @@ -63,7 +61,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock } bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { - assert(pEntry != NULL); return pEntry->complete; } diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index b0f23f78df..e7d631f638 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -73,7 +73,10 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { #if defined(USE_ARRAYLIST) int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); - assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL); + if (ASSERTS(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL, "tHistogramAdd Error, idx:%d, maxEntries:%d, elems:%p", + idx, (*pHisto)->maxEntries, (*pHisto)->elems)) { + return -1; + } if ((*pHisto)->elems[idx].val == val && idx >= 0) { (*pHisto)->elems[idx].num += 1; @@ -84,15 +87,27 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } else { /* insert a new slot */ if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { if (idx > 0) { - assert((*pHisto)->elems[idx - 1].val <= val); + if (ASSERTS((*pHisto)->elems[idx - 1].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", + idx - 1, (*pHisto)->elems[idx - 1].val, val)) { + return -1; + } } else { - assert((*pHisto)->elems[idx].val > val); + if (ASSERTS((*pHisto)->elems[idx].val > val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", + idx, (*pHisto)->elems[idx].val, val)) { + return -1; + } } } else if ((*pHisto)->numOfElems > 0) { - assert((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val); + if (ASSERTS((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", + (*pHisto)->numOfEntries, (*pHisto)->elems[idx].val, val)) { + return -1; + } } - histogramCreateBin(*pHisto, idx, val); + int32_t code = histogramCreateBin(*pHisto, idx, val); + if (code != 0) { + return code; + } } #else tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); @@ -151,7 +166,6 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { if ((*pHisto)->numOfEntries >= (*pHisto)->maxEntries + 1) { // set the right value for loser-tree - assert((*pHisto)->pLoserTree != NULL); if (!(*pHisto)->ordered) { SSkipListPrint((*pHisto)->pList, 1); @@ -203,7 +217,10 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { tSkipListNode* pNext = pNode->pForward[0]; SHistBin* pNextEntry = (SHistBin*)pNext->pData; - assert(pNextEntry->val - pEntry->val == pEntry->delta); + if (ASSERTS(pNextEntry->val - pEntry->val == pEntry->delta, "tHistogramAdd Error, pNextEntry->val:%lf, pEntry->val:%lf, pEntry->delta:%lf", + pNextEntry->val, pEntry->val, pEntry->delta)) { + return -1; + } double newVal = (pEntry->val * pEntry->num + pNextEntry->val * pNextEntry->num) / (pEntry->num + pNextEntry->num); pEntry->val = newVal; @@ -253,7 +270,9 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } else { SHistBin* pEntry = (SHistBin*)pResNode->pData; - assert(pEntry->val == val); + if (ASSERTS(pEntry->val == val, "tHistogramAdd Error, pEntry->val:%lf, val:%lf")) { + return -1; + } pEntry->num += 1; } @@ -329,7 +348,10 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); } - assert(index >= 0 && index <= pHisto->maxEntries); + if (ASSERTS(index >= 0 && index <= pHisto->maxEntries, "histogramCreateBin Error, index:%d, maxEntries:%d", + index, pHisto->maxEntries)) { + return -1; + } pHisto->elems[index].num = 1; pHisto->elems[index].val = val; @@ -343,7 +365,11 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) { pHisto->elems[pHisto->maxEntries].num = 0; } #endif - assert(pHisto->numOfEntries <= pHisto->maxEntries); + if (ASSERTS(pHisto->numOfEntries <= pHisto->maxEntries, "histogramCreateBin Error, numOfEntries:%d, maxEntries:%d", + pHisto->numOfEntries, pHisto->maxEntries)) { + return -1; + } + return 0; } @@ -386,12 +412,14 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { if (slotIdx < 0) { slotIdx = 0; - assert(v <= pHisto->elems[slotIdx].val); + ASSERTS(v <= pHisto->elems[slotIdx].val, "tHistogramSum Error, elems[%d].val:%lf, v:%lf", + slotIdx, pHisto->elems[slotIdx].val, v); } else { - assert(v >= pHisto->elems[slotIdx].val); - + ASSERTS(v >= pHisto->elems[slotIdx].val, "tHistogramSum Error, elems[%d].val:%lf, v:%lf", + slotIdx, pHisto->elems[slotIdx].val, v); if (slotIdx + 1 < pHisto->numOfEntries) { - assert(v < pHisto->elems[slotIdx + 1].val); + ASSERTS(v < pHisto->elems[slotIdx + 1].val, "tHistogramSum Error, elems[%d].val:%lf, v:%lf", + slotIdx + 1, pHisto->elems[slotIdx + 1].val, v); } } } @@ -445,7 +473,9 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { j += 1; } - assert(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem); + ASSERTS(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem, + "tHistogramUniform Error, total:%d, numOfElem:%d, elems[%d].num:%d", + total, numOfElem, j + 1, pHisto->elems[j + 1].num); double delta = numOfElem - total; if (fabs(delta) < FLT_EPSILON) { @@ -502,7 +532,9 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { j += 1; } - assert(total <= numOfElem && total + pEntry->num > numOfElem); + ASSERTS(total <= numOfElem && total + pEntry->num > numOfElem, + "tHistogramUniform Error, total:%d, numOfElem:%d, pEntry->num:%d", + total, numOfElem, pEntry->num); double delta = numOfElem - total; if (fabs(delta) < FLT_EPSILON) { diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 157ee08f15..04472c42ec 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -40,6 +40,9 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) int32_t *pageId = taosArrayGet(pIdList, i); SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); + if (pg == NULL) { + return NULL; + } memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); offset += (int32_t)(pg->num * pMemBucket->bytes); @@ -88,7 +91,7 @@ static void resetPosInfo(SSlotInfo *pInfo) { } double findOnlyResult(tMemBucket *pMemBucket) { - assert(pMemBucket->total == 1); + ASSERT(pMemBucket->total == 1); for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) { tMemBucketSlot *pSlot = &pMemBucket->pSlots[i]; @@ -100,11 +103,14 @@ double findOnlyResult(tMemBucket *pMemBucket) { SArray **pList = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); if (pList != NULL) { SArray *list = *pList; - assert(list->size == 1); + ASSERT(list->size == 1); int32_t *pageId = taosArrayGet(list, 0); SFilePage *pPage = getBufPage(pMemBucket->pBuffer, *pageId); - assert(pPage->num == 1); + if (pPage == NULL) { + return -1; + } + ASSERT(pPage->num == 1); double v = 0; GET_TYPED_DATA(v, double, pMemBucket->type, pPage->data); @@ -140,7 +146,8 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { } } - assert(index >= 0 && index < pBucket->numOfSlots); + ASSERTS(index >= 0 && index < pBucket->numOfSlots, "tBucketIntHash Error, index:%d, numOfSlots:%d", + index, pBucket->numOfSlots); return index; } @@ -167,7 +174,7 @@ int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) { } } - assert(index >= 0 && index < pBucket->numOfSlots); + ASSERT(index >= 0 && index < pBucket->numOfSlots); return index; } @@ -198,7 +205,7 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { } } - assert(index >= 0 && index < pBucket->numOfSlots); + ASSERT(index >= 0 && index < pBucket->numOfSlots); return index; } @@ -331,7 +338,7 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataT r->dMaxVal = v; } } else { - assert(0); + ASSERT(0); } } @@ -339,7 +346,7 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataT * in memory bucket, we only accept data array list */ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { - assert(pBucket != NULL && data != NULL && size > 0); + ASSERT(pBucket != NULL && data != NULL && size > 0); int32_t count = 0; int32_t bytes = pBucket->bytes; @@ -361,7 +368,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) { if (pSlot->info.data != NULL) { - assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); + ASSERT(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); // keep the pointer in memory setBufPageDirty(pSlot->info.data, true); @@ -379,6 +386,9 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { } pSlot->info.data = getNewBufPage(pBucket->pBuffer, &pageId); + if (pSlot->info.data == NULL) { + return TSDB_CODE_NO_AVAIL_DISK; + } pSlot->info.pageId = pageId; taosArrayPush(pPageIdList, &pageId); } @@ -390,7 +400,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { } pBucket->total += count; - return 0; + return TSDB_CODE_SUCCESS; } //////////////////////////////////////////////////////////////////////////////////////////// @@ -407,14 +417,14 @@ static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int3 ++j; } - assert(j < pMemBucket->numOfSlots); + ASSERT(j < pMemBucket->numOfSlots); return pMemBucket->pSlots[j].range; } static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index); static double getIdenticalDataVal(tMemBucket *pMemBucket, int32_t slotIndex) { - assert(isIdenticalData(pMemBucket, slotIndex)); + ASSERT(isIdenticalData(pMemBucket, slotIndex)); tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex]; @@ -461,7 +471,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) minOfNextSlot = (double)next.dMinVal; } - assert(minOfNextSlot > maxOfThisSlot); + ASSERT(minOfNextSlot > maxOfThisSlot); double val = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot; return val; @@ -470,6 +480,9 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) if (pSlot->info.size <= pMemBucket->maxCapacity) { // data in buffer and file are merged together to be processed. SFilePage *buffer = loadDataFromFilePage(pMemBucket, i); + if (buffer == NULL) { + return -1; + } int32_t currentIdx = count - num; char *thisVal = buffer->data + pMemBucket->bytes * currentIdx; @@ -504,8 +517,14 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) for (int32_t f = 0; f < list->size; ++f) { int32_t *pageId = taosArrayGet(list, f); SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); + if (pg == NULL) { + return -1; + } - tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); + int32_t code = tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); + if (code != TSDB_CODE_SUCCESS) { + return -1; + } setBufPageDirty(pg, true); releaseBufPage(pMemBucket->pBuffer, pg); } @@ -527,7 +546,9 @@ double getPercentile(tMemBucket *pMemBucket, double percent) { // if only one elements exists, return it if (pMemBucket->total == 1) { - return findOnlyResult(pMemBucket); + if (findOnlyResult(pMemBucket) < 0) { + return -1; + } } percent = fabs(percent); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index e51eab453c..fe6ed7d785 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -460,13 +460,14 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { #else snprintf(path, sizeof(path), "%s/lib%s.so", tsTempDir, pFuncInfo->name); #endif - TdFilePtr file = - taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); + + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (file == NULL) { fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno)); msgInfo->code = TSDB_CODE_FILE_CORRUPTED; goto _return; } + int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); if (count != pFuncInfo->codeSize) { fnError("udfd write udf shared library failed"); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c1982f3c7e..05d49bb027 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5527,7 +5527,8 @@ static void getStreamQueryFirstProjectAliasName(SHashObj* pUserAliasSet, char* a return; } -static int32_t addWstartTsToCreateStreamQueryImpl(SSelectStmt* pSelect, SHashObj* pUserAliasSet) { +static int32_t addWstartTsToCreateStreamQueryImpl(STranslateContext* pCxt, SSelectStmt* pSelect, + SHashObj* pUserAliasSet) { SNode* pProj = nodesListGetNode(pSelect->pProjectionList, 0); if (NULL == pSelect->pWindow || (QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp("_wstart", ((SFunctionNode*)pProj)->functionName))) { @@ -5539,7 +5540,10 @@ static int32_t addWstartTsToCreateStreamQueryImpl(SSelectStmt* pSelect, SHashObj } strcpy(pFunc->functionName, "_wstart"); getStreamQueryFirstProjectAliasName(pUserAliasSet, pFunc->node.aliasName, sizeof(pFunc->node.aliasName)); - int32_t code = nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); + int32_t code = getFuncInfo(pCxt, pFunc); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); + } if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pFunc); } @@ -5551,7 +5555,7 @@ static int32_t addWstartTsToCreateStreamQuery(STranslateContext* pCxt, SNode* pS SHashObj* pUserAliasSet = NULL; int32_t code = checkProjectAlias(pCxt, pSelect->pProjectionList, &pUserAliasSet); if (TSDB_CODE_SUCCESS == code) { - code = addWstartTsToCreateStreamQueryImpl(pSelect, pUserAliasSet); + code = addWstartTsToCreateStreamQueryImpl(pCxt, pSelect, pUserAliasSet); } taosHashCleanup(pUserAliasSet); return code; @@ -5664,13 +5668,13 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; - int32_t code = addWstartTsToCreateStreamQuery(pCxt, pStmt->pQuery); - if (TSDB_CODE_SUCCESS == code) { - code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt); - } + int32_t code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { code = translateQuery(pCxt, pStmt->pQuery); } + if (TSDB_CODE_SUCCESS == code) { + code = addWstartTsToCreateStreamQuery(pCxt, pStmt->pQuery); + } if (TSDB_CODE_SUCCESS == code) { code = checkStreamQuery(pCxt, pStmt); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index cf6e251d72..dcb7c02580 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -202,6 +202,15 @@ _return: QW_RET(code); } +bool qwTaskNotInExec(SQWTaskCtx *ctx) { + qTaskInfo_t taskHandle = ctx->taskHandle; + if (NULL == taskHandle || !qTaskIsExecuting(taskHandle)) { + return true; + } + + return false; +} + int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; @@ -508,8 +517,10 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp } if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { - QW_ERR_JRET(qwDropTask(QW_FPARAMS())); - QW_ERR_JRET(ctx->rspCode); + if (QW_PHASE_POST_FETCH != phase || qwTaskNotInExec(ctx)) { + QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + QW_ERR_JRET(ctx->rspCode); + } } if (ctx->rspCode) { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index d0c27560ca..de660ae958 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -90,7 +90,7 @@ rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRan int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { if (optr2) { - assert(optr2 == OP_TYPE_LOWER_THAN || optr2 == OP_TYPE_LOWER_EQUAL); + ASSERT(optr2 == OP_TYPE_LOWER_THAN || optr2 == OP_TYPE_LOWER_EQUAL); if (optr == OP_TYPE_GREATER_THAN) { if (optr2 == OP_TYPE_LOWER_THAN) { @@ -705,7 +705,7 @@ int32_t filterAddRangeImpl(void *h, SFilterRange *ra, int32_t optr) { int32_t filterAddRange(void *h, SFilterRange *ra, int32_t optr) { SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; int64_t tmp = 0; - + if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type, &tmp)); // FILTER_CLR_FLAG(ra->sflag, RA_NULL); @@ -723,7 +723,7 @@ int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - assert(optr == LOGIC_COND_TYPE_OR); + ASSERT(optr == LOGIC_COND_TYPE_OR); if (sctx->rs == NULL) { return TSDB_CODE_SUCCESS; @@ -778,7 +778,10 @@ int32_t filterFinishRange(void *h) { while (r && r->next) { int64_t tmp = 1; - operateVal(&tmp, &r->ra.e, &tmp, OP_TYPE_ADD, ctx->type); + int32_t code = operateVal(&tmp, &r->ra.e, &tmp, OP_TYPE_ADD, ctx->type); + if (code != 0) { + return TSDB_CODE_APP_ERROR; + } if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { rn = r->next; SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); @@ -1122,7 +1125,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, if (u->right.type == FLD_TYPE_VALUE) { SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); - assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); + ASSERT(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); } else { int32_t paramNum = scalarGetOperatorParamNum(optr); if (1 != paramNum) { @@ -1132,7 +1135,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, } SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); - assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); + ASSERT(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col); info->units[info->unitNum].compare.precision = FILTER_GET_COL_FIELD_PRECISION(col); @@ -1292,29 +1295,29 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (optr == LOGIC_COND_TYPE_AND) { if (ctx->isnull) { - assert(ctx->notnull == false && ctx->isrange == false); + ASSERT(ctx->notnull == false && ctx->isrange == false); filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } if (ctx->notnull) { - assert(ctx->isnull == false && ctx->isrange == false); + ASSERT(ctx->isnull == false && ctx->isrange == false); filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } if (!ctx->isrange) { - assert(ctx->isnull || ctx->notnull); + ASSERT(ctx->isnull || ctx->notnull); return TSDB_CODE_SUCCESS; } - assert(ctx->rs && ctx->rs->next == NULL); + ASSERT(ctx->rs && ctx->rs->next == NULL); SFilterRange *ra = &ctx->rs->ra; - assert(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); + ASSERT(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); @@ -1368,7 +1371,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SFilterGroup ng = {0}; g = &ng; - assert(ctx->isnull || ctx->notnull || ctx->isrange); + ASSERT(ctx->isnull || ctx->notnull || ctx->isrange); if (ctx->isnull) { filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); @@ -1377,7 +1380,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (ctx->notnull) { - assert(!ctx->isrange); + ASSERT(!ctx->isrange); memset(g, 0, sizeof(*g)); filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); @@ -1386,7 +1389,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!ctx->isrange) { - assert(ctx->isnull || ctx->notnull); + ASSERT(ctx->isnull || ctx->notnull); g->unitNum = 0; return TSDB_CODE_SUCCESS; } @@ -1444,7 +1447,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan filterAddUnitToGroup(g, uidx); } - assert(g->unitNum > 0); + ASSERT(g->unitNum > 0); taosArrayPush(res, g); @@ -1900,7 +1903,7 @@ void filterFreeInfo(SFilterInfo *info) { } int32_t filterHandleValueExtInfo(SFilterUnit *unit, char extInfo) { - assert(extInfo > 0 || extInfo < 0); + ASSERT(extInfo > 0 || extInfo < 0); uint8_t optr = FILTER_UNIT_OPTR(unit); switch (optr) { @@ -1916,7 +1919,8 @@ int32_t filterHandleValueExtInfo(SFilterUnit *unit, char extInfo) { unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; break; default: - assert(0); + fltError("unsupported operator type"); + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -1926,13 +1930,13 @@ int32_t fltInitValFieldData(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { - assert(unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR || scalarGetOperatorParamNum(unit->compare.optr) == 1); + ASSERT(unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR || scalarGetOperatorParamNum(unit->compare.optr) == 1); continue; } SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - assert(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); + ASSERT(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); uint32_t type = FILTER_UNIT_DATA_TYPE(unit); int8_t precision = FILTER_UNIT_DATA_PRECISION(unit); @@ -1940,7 +1944,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { SValueNode *var = (SValueNode *)fi->desc; if (var == NULL) { - assert(fi->data != NULL); + ASSERT(fi->data != NULL); continue; } @@ -2068,7 +2072,8 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) } default: - assert(false); + fltError("unsupported operator type"); + return false; } return true; @@ -2101,7 +2106,7 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit *u, SFilterRangeCtx *c FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); break; case OP_TYPE_NOT_EQUAL: - assert(type == TSDB_DATA_TYPE_BOOL); + ASSERT(type == TSDB_DATA_TYPE_BOOL); if (GET_INT8_VAL(val)) { SIMPLE_COPY_VALUES(&ra.s, &tmp); SIMPLE_COPY_VALUES(&ra.e, &tmp); @@ -2116,7 +2121,8 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit *u, SFilterRangeCtx *c SIMPLE_COPY_VALUES(&ra.e, val); break; default: - assert(0); + fltError("unsupported operator type"); + return TSDB_CODE_APP_ERROR; } filterAddRange(ctx, &ra, optr); @@ -2368,8 +2374,8 @@ int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32 filterReuseRangeCtx(*ctx, type, 0); } - assert(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); - assert(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + ASSERT(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + ASSERT(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); filterCopyRangeCtx(*ctx, gRes2->colInfo[cidx].info); filterSourceRangeFromCtx(*ctx, gRes1->colInfo[cidx].info, optr, empty, all); @@ -2405,7 +2411,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx **gRes1, SFilter continue; } - assert(idx1 == idx2); + ASSERT(idx1 == idx2); ++merNum; @@ -2455,15 +2461,15 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx **gRes1, SFilter } } - assert(merNum > 0); + ASSERT(merNum > 0); SFilterColInfo *colInfo = NULL; - assert(merNum == equal1 || merNum == equal2); + ASSERT(merNum == equal1 || merNum == equal2); filterFreeGroupCtx(*gRes2); *gRes2 = NULL; - assert(colCtxs && taosArrayGetSize(colCtxs) > 0); + ASSERT(colCtxs && taosArrayGetSize(colCtxs) > 0); int32_t ctxSize = (int32_t)taosArrayGetSize(colCtxs); SFilterColCtx *pctx = NULL; @@ -2520,7 +2526,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t *gR if (pColNum > 0) { for (int32_t m = 0; m <= pEnd; ++m) { for (int32_t n = cStart; n <= cEnd; ++n) { - assert(m < n); + ASSERT(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); FLT_CHK_JMP(all); @@ -2541,7 +2547,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t *gR for (int32_t m = cStart; m < cEnd; ++m) { for (int32_t n = m + 1; n <= cEnd; ++n) { - assert(m < n); + ASSERT(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); FLT_CHK_JMP(all); @@ -2636,7 +2642,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum for (uint32_t m = 0; m < res->colNum; ++m) { colInfo = &res->colInfo[res->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { - assert(colInfo->type == RANGE_TYPE_UNIT); + ASSERT(colInfo->type == RANGE_TYPE_UNIT); int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); for (int32_t n = 0; n < usize; ++n) { @@ -2649,7 +2655,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum continue; } - assert(colInfo->type == RANGE_TYPE_MR_CTX); + ASSERT(colInfo->type == RANGE_TYPE_MR_CTX); filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group); } @@ -2690,7 +2696,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx **gRes, int32_ continue; } - assert(idxNum[i] == gResNum); + ASSERT(idxNum[i] == gResNum); if (idxs == NULL) { idxs = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); @@ -2714,7 +2720,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx **gRes, int32_ continue; } - assert(res->colIdx[n] == idxs[m]); + ASSERT(res->colIdx[n] == idxs[m]); SFilterColInfo *colInfo = &res->colInfo[res->colIdx[n]]; if (info->colRange[m] == NULL) { @@ -2723,7 +2729,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx **gRes, int32_ info->colRange[m]->colId = FILTER_GET_COL_FIELD_ID(fi); } - assert(colInfo->type == RANGE_TYPE_MR_CTX); + ASSERT(colInfo->type == RANGE_TYPE_MR_CTX); bool all = false; filterSourceRangeFromCtx(info->colRange[m], colInfo->info, LOGIC_COND_TYPE_OR, NULL, &all); @@ -2971,7 +2977,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 unitIdx = pGroupIdx; --info->blkGroupNum; - assert(empty || all); + ASSERT(empty || all); if (empty) { FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); @@ -3077,7 +3083,7 @@ int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, SColumn goto _return; } - assert(info->unitNum > 1); + ASSERT(info->unitNum > 1); *all = filterExecuteBasedOnStatisImpl(info, numOfRows, p, statis, numOfCols); goto _return; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index debc210f0f..c1e958b055 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -327,7 +327,10 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; - ASSERT(param->columnData == NULL); + if (param->columnData != NULL) { + sclError("columnData should be NULL"); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } param->numOfRows = 1; int32_t code = sclCreateColumnInfoData(&valueNode->node.resType, 1, param); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 1de8a35308..411b9b7ab5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -361,7 +361,6 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - ASSERT(pOutputData->info.type == TSDB_DATA_TYPE_BIGINT); int64_t *out = (int64_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -1729,37 +1728,31 @@ bool getTimePseudoFuncEnv(SFunctionNode *UNUSED_PARAM(pFunc), SFuncExecEnv *pEnv } int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0)); return TSDB_CODE_SUCCESS; } int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 1)); return TSDB_CODE_SUCCESS; } int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 2)); return TSDB_CODE_SUCCESS; } int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 3)); return TSDB_CODE_SUCCESS; } int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 4)); return TSDB_CODE_SUCCESS; } int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - ASSERT(inputNum == 1); char* p = colDataGetVarData(pInput->columnData, 0); colDataAppendNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows); @@ -2598,7 +2591,7 @@ static bool checkStateOp(int8_t op, SColumnInfoData *pCol, int32_t index, SScala break; } default: { - ASSERT(0); + return false; } } return false; @@ -2771,7 +2764,9 @@ static bool getHistogramBinDesc(SHistoFuncBin **bins, int32_t *binNum, char *bin intervals[0] = -INFINITY; intervals[numOfBins - 1] = INFINITY; // in case of desc bin orders, -inf/inf should be swapped - ASSERT(numOfBins >= 4); + if (numOfBins < 4) { + return false; + } if (intervals[1] > intervals[numOfBins - 2]) { TSWAP(intervals[0], intervals[numOfBins - 1]); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 8f2fe87a53..e9f345215e 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -389,18 +389,18 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { func = varToUnsigned; } else if (IS_FLOAT_TYPE(pCtx->outType)) { func = varToFloat; - } else if (pCtx->outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary - ASSERT(pCtx->inType == TSDB_DATA_TYPE_NCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_VARCHAR && + pCtx->inType == TSDB_DATA_TYPE_NCHAR) { // nchar -> binary func = ncharToVar; vton = true; - } else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar - ASSERT(pCtx->inType == TSDB_DATA_TYPE_VARCHAR); + } else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR && + pCtx->inType == TSDB_DATA_TYPE_VARCHAR) { // binary -> nchar func = varToNchar; vton = true; } else if (TSDB_DATA_TYPE_TIMESTAMP == pCtx->outType) { func = varToTimestamp; } else { - sclError("invalid convert outType:%d", pCtx->outType); + sclError("invalid convert outType:%d, inType:%d", pCtx->outType, pCtx->inType); return TSDB_CODE_APP_ERROR; } @@ -416,12 +416,10 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { char *data = colDataGetVarData(pCtx->pIn->columnData, i); int32_t convertType = pCtx->inType; if (pCtx->inType == TSDB_DATA_TYPE_JSON) { - if (*data == TSDB_DATA_TYPE_NULL) { - ASSERT(0); - } else if (*data == TSDB_DATA_TYPE_NCHAR) { + if (*data == TSDB_DATA_TYPE_NCHAR) { data += CHAR_BYTES; convertType = TSDB_DATA_TYPE_NCHAR; - } else if (tTagIsJson(data)) { + } else if (tTagIsJson(data) || *data == TSDB_DATA_TYPE_NULL) { terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR; return terrno; } else { @@ -447,7 +445,11 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { tmp[varDataLen(data)] = 0; } else if (TSDB_DATA_TYPE_NCHAR == convertType) { // we need to convert it to native char string, and then perform the string to numeric data - ASSERT(varDataLen(data) <= bufSize); + if (varDataLen(data) > bufSize) { + sclError("castConvert convert buffer size too small"); + taosMemoryFreeClear(tmp); + return TSDB_CODE_APP_ERROR; + } int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); if (len < 0) { @@ -557,27 +559,17 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *fp = filterGetCompFunc(type, optr); if (IS_NUMERIC_TYPE(type)) { - if (typeLeft == TSDB_DATA_TYPE_NCHAR) { - ASSERT(0); - // convertNcharToDouble(*pLeftData, pLeftOut); - // *pLeftData = pLeftOut; - } else if (typeLeft == TSDB_DATA_TYPE_BINARY) { - ASSERT(0); - // convertBinaryToDouble(*pLeftData, pLeftOut); - // *pLeftData = pLeftOut; + if (typeLeft == TSDB_DATA_TYPE_NCHAR || + typeLeft == TSDB_DATA_TYPE_VARCHAR) { + return false; } else if (typeLeft != type) { convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); *pLeftData = pLeftOut; } - if (typeRight == TSDB_DATA_TYPE_NCHAR) { - ASSERT(0); - // convertNcharToDouble(*pRightData, pRightOut); - // *pRightData = pRightOut; - } else if (typeRight == TSDB_DATA_TYPE_BINARY) { - ASSERT(0); - // convertBinaryToDouble(*pRightData, pRightOut); - // *pRightData = pRightOut; + if (typeRight == TSDB_DATA_TYPE_NCHAR || + typeRight == TSDB_DATA_TYPE_VARCHAR) { + return false; } else if (typeRight != type) { convertNumberToNumber(*pRightData, pRightOut, typeRight, type); *pRightData = pRightOut; @@ -592,7 +584,7 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *freeRight = true; } } else { - ASSERT(0); + return false; } return true; @@ -683,7 +675,10 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } if (overflow) { - ASSERT(1 == pIn->numOfRows); + if (1 != pIn->numOfRows) { + sclError("invalid numOfRows %d", pIn->numOfRows); + return TSDB_CODE_APP_ERROR; + } pOut->numOfRows = 0; @@ -1938,7 +1933,6 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { case OP_TYPE_JSON_CONTAINS: return vectorJsonContains; default: - ASSERT(0); return NULL; } } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3bf4a8d1cd..6793430923 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -228,6 +228,7 @@ int32_t syncNodeStart(SSyncNode* pSyncNode); int32_t syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); void syncNodePreClose(SSyncNode* pSyncNode); +void syncNodePostClose(SSyncNode* pSyncNode); int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t *seq); int32_t syncNodeRestore(SSyncNode* pSyncNode); void syncHbTimerDataFree(SSyncHbTimerData* pData); diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 974a8f968e..5277e7818f 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -57,7 +57,6 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender); bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); int32_t snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish); -int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); typedef struct SSyncSnapshotReceiver { @@ -82,7 +81,6 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver); // on message int32_t syncNodeOnSnapshot(SSyncNode *ths, const SRpcMsg *pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b944be0c67..91a05e78b0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -124,6 +124,14 @@ void syncPreStop(int64_t rid) { } } +void syncPostStop(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode != NULL) { + syncNodePostClose(pSyncNode); + syncNodeRelease(pSyncNode); + } +} + static bool syncNodeCheckNewConfig(SSyncNode* pSyncNode, const SSyncCfg* pCfg) { if (!syncNodeInConfig(pSyncNode, pCfg)) return false; return abs(pCfg->replicaNum - pSyncNode->replicaNum) <= 1; @@ -851,6 +859,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { if (!taosCheckExistFile(pSyncNode->configPath)) { // create a new raft config file + sInfo("vgId:%d, create a new raft config file", pSyncNode->vgId); pSyncNode->raftCfg.isStandBy = pSyncInfo->isStandBy; pSyncNode->raftCfg.snapshotStrategy = pSyncInfo->snapshotStrategy; pSyncNode->raftCfg.lastConfigIndex = SYNC_INDEX_INVALID; @@ -894,7 +903,6 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pNode->nodeId, pNode->clusterId); } - pSyncNode->pWal = pSyncInfo->pWal; pSyncNode->msgcb = pSyncInfo->msgcb; pSyncNode->syncSendMSg = pSyncInfo->syncSendMSg; @@ -1236,9 +1244,10 @@ void syncNodePreClose(SSyncNode* pSyncNode) { } } +#if 0 if (pSyncNode->pNewNodeReceiver != NULL) { if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { - snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); + snapshotReceiverStop(pSyncNode->pNewNodeReceiver); } sDebug("vgId:%d, snapshot receiver destroy while preclose sync node, data:%p", pSyncNode->vgId, @@ -1246,12 +1255,29 @@ void syncNodePreClose(SSyncNode* pSyncNode) { snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); pSyncNode->pNewNodeReceiver = NULL; } +#endif // stop elect timer syncNodeStopElectTimer(pSyncNode); // stop heartbeat timer syncNodeStopHeartbeatTimer(pSyncNode); + + // clean rsp + syncRespCleanRsp(pSyncNode->pSyncRespMgr); +} + +void syncNodePostClose(SSyncNode* pSyncNode) { + if (pSyncNode->pNewNodeReceiver != NULL) { + if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { + snapshotReceiverStop(pSyncNode->pNewNodeReceiver); + } + + sDebug("vgId:%d, snapshot receiver destroy while preclose sync node, data:%p", pSyncNode->vgId, + pSyncNode->pNewNodeReceiver); + snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); + pSyncNode->pNewNodeReceiver = NULL; + } } void syncHbTimerDataFree(SSyncHbTimerData* pData) { taosMemoryFree(pData); } @@ -1299,7 +1325,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { if (pSyncNode->pNewNodeReceiver != NULL) { if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { - snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); + snapshotReceiverStop(pSyncNode->pNewNodeReceiver); } sDebug("vgId:%d, snapshot receiver destroy while close, data:%p", pSyncNode->vgId, pSyncNode->pNewNodeReceiver); @@ -1656,7 +1682,6 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // persist cfg syncWriteCfgFile(pSyncNode); - // change isStandBy to normal (election timeout) if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeBecomeLeader(pSyncNode, ""); @@ -1830,7 +1855,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // close receiver if (pSyncNode != NULL && pSyncNode->pNewNodeReceiver != NULL && snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { - snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); + snapshotReceiverStop(pSyncNode->pNewNodeReceiver); } // stop elect timer @@ -2668,16 +2693,12 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn } int32_t code = syncNodeAppend(ths, pEntry); - if (code < 0) { - sNError(ths, "failed to append blocking msg"); - } return code; } else { syncEntryDestroy(pEntry); pEntry = NULL; + return -1; } - - return -1; } int32_t syncNodeOnClientRequestOld(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) { diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index b68b735f46..defb7402f4 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -150,7 +150,7 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { // when sender receive ack, call this function to send msg from seq // seq = ack + 1, already updated -int32_t snapshotSend(SSyncSnapshotSender *pSender) { +static int32_t snapshotSend(SSyncSnapshotSender *pSender) { // free memory last time (current seq - 1) if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); @@ -342,23 +342,6 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } -// force stop -void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { - sRInfo(pReceiver, "snapshot receiver force stop, writer:%p"); - - // force close, abandon incomplete data - if (pReceiver->pWriter != NULL) { - int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, - &pReceiver->snapshot); - if (ret != 0) { - sRInfo(pReceiver, "snapshot receiver force stop failed since %s", terrstr()); - } - pReceiver->pWriter = NULL; - } - - pReceiver->start = false; -} - static int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { if (pReceiver->pWriter != NULL) { sRError(pReceiver, "vgId:%d, snapshot receiver writer is not null"); @@ -590,7 +573,7 @@ _START_RECEIVER: if (snapshotReceiverIsStart(pReceiver)) { sRInfo(pReceiver, "snapshot receiver already start and force stop pre one"); - snapshotReceiverForceStop(pReceiver); + snapshotReceiverStop(pReceiver); } snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender @@ -842,7 +825,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { // force close, no response syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process force stop"); - snapshotReceiverForceStop(pReceiver); + snapshotReceiverStop(pReceiver); } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq data"); code = syncNodeOnSnapshotReceive(pSyncNode, pMsg); @@ -989,6 +972,13 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { return syncNodeOnSnapshotPreRsp(pSyncNode, pSender, pMsg); } + if (pSender->pReader == NULL || pSender->finish) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender invalid"); + sSError(pSender, "snapshot sender invalid, pReader:%p finish:%d", pMsg->code, pSender->pReader, pSender->finish); + terrno = pMsg->code; + goto _ERROR; + } + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_BEGIN) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "process seq begin"); if (snapshotSenderUpdateProgress(pSender, pMsg) != 0) { diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 5d9b6ee5cc..3aac5e9775 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -59,4 +59,8 @@ endif() IF (JEMALLOC_ENABLED) target_link_libraries(os PUBLIC -ljemalloc) -ENDIF () \ No newline at end of file +ENDIF () + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index d8cccc83ed..3f527e9130 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -39,14 +39,6 @@ #define _SEND_FILE_STEP_ 1000 #endif -#if defined(WINDOWS) -typedef int32_t FileFd; -typedef int32_t SocketFd; -#else -typedef int32_t FileFd; -typedef int32_t SocketFd; -#endif - typedef int32_t FileFd; typedef struct TdFile { @@ -54,19 +46,10 @@ typedef struct TdFile { int refId; FileFd fd; FILE *fp; -} * TdFilePtr, TdFile; +} TdFile; #define FILE_WITH_LOCK 1 -typedef struct AutoDelFile *AutoDelFilePtr; -typedef struct AutoDelFile { - char *name; - AutoDelFilePtr lastAutoDelFilePtr; -} AutoDelFile; -static TdThreadMutex autoDelFileLock; -static AutoDelFilePtr nowAutoDelFilePtr = NULL; -static TdThreadOnce autoDelFileInit = PTHREAD_ONCE_INIT; - void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { #ifdef WINDOWS const char *tdengineTmpFileNamePrefix = "tdengine-"; @@ -268,34 +251,6 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { return 0; } -void autoDelFileList() { - taosThreadMutexLock(&autoDelFileLock); - while (nowAutoDelFilePtr != NULL) { - taosRemoveFile(nowAutoDelFilePtr->name); - AutoDelFilePtr tmp = nowAutoDelFilePtr->lastAutoDelFilePtr; - taosMemoryFree(nowAutoDelFilePtr->name); - taosMemoryFree(nowAutoDelFilePtr); - nowAutoDelFilePtr = tmp; - } - taosThreadMutexUnlock(&autoDelFileLock); - taosThreadMutexDestroy(&autoDelFileLock); -} - -void autoDelFileListInit() { - taosThreadMutexInit(&autoDelFileLock, NULL); - atexit(autoDelFileList); -} - -void autoDelFileListAdd(const char *path) { - taosThreadOnce(&autoDelFileInit, autoDelFileListInit); - taosThreadMutexLock(&autoDelFileLock); - AutoDelFilePtr tmp = taosMemoryMalloc(sizeof(AutoDelFile)); - tmp->lastAutoDelFilePtr = nowAutoDelFilePtr; - tmp->name = taosMemoryStrDup(path); - nowAutoDelFilePtr = tmp; - taosThreadMutexUnlock(&autoDelFileLock); -} - TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { int fd = -1; FILE *fp = NULL; @@ -313,7 +268,6 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { assert(!(tdFileOptions & TD_FILE_EXCL)); fp = fopen(path, mode); if (fp == NULL) { - // terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } } else { @@ -331,32 +285,44 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; #ifdef WINDOWS - fd = _open(path, access, _S_IREAD | _S_IWRITE); + int32_t pmode = _S_IREAD | _S_IWRITE; + if (tdFileOptions & TD_FILE_AUTO_DEL) { + pmode |= _O_TEMPORARY; + } + fd = _open(path, access, pmode); #else fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); #endif if (fd == -1) { - // terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } } TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile)); if (pFile == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; if (fd >= 0) close(fd); if (fp != NULL) fclose(fp); return NULL; } + #if FILE_WITH_LOCK taosThreadRwlockInit(&(pFile->rwlock), NULL); #endif pFile->fd = fd; pFile->fp = fp; pFile->refId = 0; + if (tdFileOptions & TD_FILE_AUTO_DEL) { - autoDelFileListAdd(path); +#ifdef WINDOWS + // do nothing, since the property of pmode is set with _O_TEMPORARY; the OS will recycle + // the file handle, as well as the space on disk. +#else + // Remove it instantly, so when the program exits normally/abnormally, the file + // will be automatically remove by OS. + unlink(path); +#endif } + return pFile; } @@ -560,6 +526,21 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { int32_t taosLockFile(TdFilePtr pFile) { #ifdef WINDOWS + BOOL fSuccess = FALSE; + LARGE_INTEGER fileSize; + OVERLAPPED overlapped = {0}; + + HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); + + fSuccess = LockFileEx(hFile, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, + 0, // reserved + ~0, // number of bytes to lock low + ~0, // number of bytes to lock high + &overlapped // overlapped structure + ); + if (!fSuccess) { + return GetLastError(); + } return 0; #else assert(pFile->fd >= 0); // Please check if you have closed the file. @@ -570,6 +551,14 @@ int32_t taosLockFile(TdFilePtr pFile) { int32_t taosUnLockFile(TdFilePtr pFile) { #ifdef WINDOWS + BOOL fSuccess = FALSE; + OVERLAPPED overlapped = {0}; + HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); + + fSuccess = UnlockFileEx(hFile, 0, ~0, ~0, &overlapped); + if (!fSuccess) { + return GetLastError(); + } return 0; #else assert(pFile->fd >= 0); // Please check if you have closed the file. diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index d79e1934de..f831f457f9 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -27,6 +27,7 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include "os.h" +#include "tlog.h" TEST(osTest, osSystem) { const char *flags = "UTL FATAL "; @@ -35,4 +36,97 @@ TEST(osTest, osSystem) { taosPrintTrace(flags, level, dflag); } +void fileOperateOnFree(void *param) { + char * fname = (char *)param; + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); + printf("On free thread open file\n"); + ASSERT_NE(pFile, nullptr); + + int ret = taosLockFile(pFile); + printf("On free thread lock file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + ret = taosUnLockFile(pFile); + printf("On free thread unlock file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + ret = taosCloseFile(&pFile); + ASSERT_EQ(ret, 0); + printf("On free thread close file ret:%d\n", ret); +} +void *fileOperateOnFreeThread(void *param) { + fileOperateOnFree(param); + return NULL; +} +void fileOperateOnBusy(void *param) { + char * fname = (char *)param; + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); + printf("On busy thread open file\n"); + ASSERT_NE(pFile, nullptr); + + int ret = taosLockFile(pFile); + printf("On busy thread lock file ret:%d\n", ret); + ASSERT_NE(ret, 0); + + ret = taosUnLockFile(pFile); + printf("On busy thread unlock file ret:%d\n", ret); +#ifdef _TD_DARWIN_64 + ASSERT_EQ(ret, 0); +#else + ASSERT_NE(ret, 0); +#endif + + ret = taosCloseFile(&pFile); + printf("On busy thread close file ret:%d\n", ret); + ASSERT_EQ(ret, 0); +} +void *fileOperateOnBusyThread(void *param) { + fileOperateOnBusy(param); + return NULL; +} + +TEST(osTest, osFile) { + char *fname = "./osfiletest1.txt"; + + TdFilePtr pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + ASSERT_NE(pOutFD, nullptr); + printf("create file success\n"); + + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); + printf("open file\n"); + ASSERT_NE(pFile, nullptr); + + int ret = taosLockFile(pFile); + printf("lock file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + + TdThread thread1, thread2; + taosThreadCreate(&(thread1), &thattr, fileOperateOnBusyThread, (void *)fname); + taosThreadAttrDestroy(&thattr); + + taosThreadJoin(thread1, NULL); + taosThreadClear(&thread1); + + ret = taosUnLockFile(pFile); + printf("unlock file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + ret = taosCloseFile(&pFile); + printf("close file ret:%d\n", ret); + ASSERT_EQ(ret, 0); + + taosThreadCreate(&(thread2), &thattr, fileOperateOnFreeThread, (void *)fname); + taosThreadAttrDestroy(&thattr); + + taosThreadJoin(thread2, NULL); + taosThreadClear(&thread2); + + //int ret = taosRemoveFile(fname); + //ASSERT_EQ(ret, 0); + //printf("remove file success"); +} + #pragma GCC diagnostic pop diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 381ef11804..7a52991e81 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -23,6 +23,7 @@ #include "thash.h" #include "tlog.h" #include "types.h" +#include "osString.h" int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -208,16 +209,16 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { int32_t len1 = varDataLen(pLeft); int32_t len2 = varDataLen(pRight); - if (len1 != len2) { - return len1 > len2 ? 1 : -1; - } else { - int32_t ret = memcmp((TdUcs4 *)pLeft, (TdUcs4 *)pRight, len1); - if (ret == 0) { + int32_t ret = tasoUcs4Compare((TdUcs4 *)varDataVal(pLeft), (TdUcs4 *)varDataVal(pRight), len1>len2 ? len2:len1); + if (ret == 0) { + if (len1 > len2) + return 1; + else if(len1 < len2) + return -1; + else return 0; - } else { - return ret > 0 ? 1 : -1; - } } + return (ret < 0) ? -1 : 1; } int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6f8b0d8e04..14c3df38a7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -276,8 +276,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST, "Subcribe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_OFFSET_NOT_EXIST, "Offset not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED, "Topic subscribed cannot be dropped") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED, "Topic must be dropped first") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED, "Topic must be dropped first") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SUB_OPTION, "Invalid subscribe option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_IN_REBALANCE, "Topic being rebalanced") // mnode-stream diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index d5aa560eb0..1205da31b3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -175,6 +175,7 @@ ,,y,script,./test.sh -f tsim/query/session.sim ,,y,script,./test.sh -f tsim/query/udf.sim ,,y,script,./test.sh -f tsim/query/udf_with_const.sim +,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim @@ -617,6 +618,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py @@ -699,8 +702,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAlterSchema.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py @@ -715,11 +718,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py @@ -832,6 +835,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 @@ -928,6 +932,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4 @@ -1034,6 +1039,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py diff --git a/tests/script/api/dbTableRoute.c b/tests/script/api/dbTableRoute.c index 2cf721875a..bbf17c41e4 100644 --- a/tests/script/api/dbTableRoute.c +++ b/tests/script/api/dbTableRoute.c @@ -26,7 +26,10 @@ #include #include "taos.h" -int rtTables = 20; +#define RT_TABLE_NUM 100 + +int rtTables = RT_TABLE_NUM; +int rtTableUs[RT_TABLE_NUM] = {0}; char hostName[128]; static void rtExecSQL(TAOS *taos, char *command) { @@ -101,6 +104,22 @@ int rtPrepare(TAOS ** p, int prefix, int suffix) { return 0; } +int32_t rtGetTimeOfDay(struct timeval *tv) { + return gettimeofday(tv, NULL); +} +static int64_t rtGetTimestampMs() { + struct timeval systemTime; + rtGetTimeOfDay(&systemTime); + return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_usec/1000; +} + +static int64_t rtGetTimestampUs() { + struct timeval systemTime; + rtGetTimeOfDay(&systemTime); + return (int64_t)systemTime.tv_sec * 1000000LL + (int64_t)systemTime.tv_usec; +} + + int rtGetDbRouteInfo(TAOS * taos) { TAOS_DB_ROUTE_INFO dbInfo; int code = taos_get_db_route_info(taos, "db1", &dbInfo); @@ -126,7 +145,10 @@ int rtGetTableRouteInfo(TAOS * taos) { char sql[1024] = {0}; for (int32_t i = 0; i < rtTables; ++i) { sprintf(table, "tb%d", i); + int64_t startTs = rtGetTimestampUs(); int code = taos_get_table_vgId(taos, "db1", table, &vgId1); + int64_t endTs = rtGetTimestampUs(); + rtTableUs[i] = (int)(endTs - startTs); if (code) { rtExit("taos_get_table_vgId", taos_errstr(NULL)); } @@ -142,9 +164,61 @@ int rtGetTableRouteInfo(TAOS * taos) { } } + printf("table vgId use us:"); + + for (int32_t i = 0; i < rtTables; ++i) { + printf("%d ", rtTableUs[i]); + } + + printf("\n"); + return 0; } +int rtGetTablesRouteInfo(TAOS * taos) { + char *table = {0}; + int *vgId1 = malloc(rtTables * sizeof(int)); + int vgId2 = 0; + char sql[1024] = {0}; + const char *tbs[RT_TABLE_NUM] = {0}; + + for (int32_t i = 0; i < rtTables; ++i) { + table = malloc(10); + sprintf(table, "tb%d", i); + tbs[i] = table; + } + + int64_t startTs = rtGetTimestampUs(); + int code = taos_get_tables_vgId(taos, "db1", tbs, rtTables, vgId1); + int64_t endTs = rtGetTimestampUs(); + rtTableUs[0] = (int)(endTs - startTs); + if (code) { + rtExit("taos_get_tables_vgId", taos_errstr(NULL)); + } + + for (int32_t i = 0; i < rtTables; ++i) { + sprintf(sql, "select vgroup_id from information_schema.ins_tables where table_name=\"tb%d\"", i); + + rtFetchVgId(taos, sql, &vgId2); + if (vgId1[i] != vgId2) { + fprintf(stderr, "!!!! table tb%d vgId mis-match, vgId(api):%d, vgId(sys):%d\n", i, vgId1[i], vgId2); + exit(1); + } else { + printf("table tb%d vgId %d\n", i, vgId1[i]); + } + } + + printf("tables vgId use us:%d\n", rtTableUs[0]); + + for (int32_t i = 0; i < rtTables; ++i) { + free((void*)tbs[i]); + } + free(vgId1); + + return 0; +} + + void rtClose(TAOS * taos) { taos_close(taos); } @@ -170,6 +244,16 @@ int rtRunCase2(void) { return 0; } +int rtRunCase3(void) { + TAOS *taos = NULL; + rtPrepare(&taos, 0, 0); + rtGetTablesRouteInfo(taos); + rtClose(taos); + + return 0; +} + + int main(int argc, char *argv[]) { if (argc != 2) { printf("usage: %s server-ip\n", argv[0]); @@ -182,6 +266,7 @@ int main(int argc, char *argv[]) { rtRunCase1(); rtRunCase2(); + rtRunCase3(); int32_t l = 5; while (l) { diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim new file mode 100644 index 0000000000..045e908a57 --- /dev/null +++ b/tests/script/tsim/query/sys_tbname.sim @@ -0,0 +1,89 @@ + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database sys_tbname; +sql use sys_tbname; +sql create stable st(ts timestamp, f int) tags(t int); +sql create table ct1 using st tags(1); +sql create table ct2 using st tags(2); + +sql create table t (ts timestamp, f int); +sql insert into t values(now, 1)(now+1s, 2); + + +sql create table t2 (ts timestamp, f1 int, f2 int); +sql insert into t2 values(now, 0, 0)(now+1s, 1, 1); + +sql select tbname from information_schema.ins_databases; +print $rows $data00 +if $rows != 3 then + return -1 +endi +if $data00 != @ins_databases@ then + return -1 +endi +sql select distinct tbname from information_schema.ins_databases; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_databases@ then + return -1 +endi + +sql select tbname from information_schema.ins_stables; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_stables@ then + return -1 +endi +sql select distinct tbname from information_schema.ins_stables; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_stables@ then + return -1 +endi + +sql select tbname from information_schema.ins_tables; +print $rows $data00 +if $rows != 32 then + return -1 +endi +if $data00 != @ins_tables@ then + return -1 +endi + +sql select distinct tbname from information_schema.ins_tables; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_tables@ then + return -1 +endi + +sql select tbname from information_schema.ins_tags; +print $rows $data00 +if $rows != 2 then + return -1 +endi +if $data00 != @ins_tags@ then + return -1 +endi +sql select distinct tbname from information_schema.ins_tags; +print $rows $data00 +if $rows != 1 then + return -1 +endi +if $data00 != @ins_tags@ then + return -1 +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/basic3.sim b/tests/script/tsim/stream/basic3.sim index 41e19b19af..e598919e34 100644 --- a/tests/script/tsim/stream/basic3.sim +++ b/tests/script/tsim/stream/basic3.sim @@ -1,6 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c debugflag 131 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/cfg.sh -n dnode1 -c keepColumnName -v 1 system sh/exec.sh -n dnode1 -s start sleep 5000 @@ -9,7 +10,7 @@ sql connect print ========== interval\session\state window -sql CREATE DATABASE test1 BUFFER 96 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 STRICT 'off' WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0; +sql CREATE DATABASE test1 VGROUPS 2; sql use test1; sql CREATE STABLE st (time TIMESTAMP, ca DOUBLE, cb DOUBLE, cc int) TAGS (ta VARCHAR(10) ); @@ -29,6 +30,76 @@ sql create stream streamd4 into streamt4 as select tbname, _wstart,_wend, count( sql create stream streamd5 into streamt5 as select tbname, _wstart,_wend, count(*), max(ca), min(cb) from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by tbname state_window(cc); sql create stream streamd6 into streamt6 as select ca, _wstart,_wend, count(*), max(ca), min(cb) from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca state_window(cc); + +sql alter local 'keepColumnName' '1' + +sql CREATE STABLE `meters_test_data` (`ts` TIMESTAMP, `close` FLOAT, `parttime` TIMESTAMP, `parttime_str` VARCHAR(32)) TAGS (`id` VARCHAR(32)); + +sql_error create stream realtime_meters fill_history 1 into realtime_meters as select last(parttime),first(close),last(close) from meters_test_data partition by tbname state_window(parttime_str); +sql_error create stream streamd7 into streamt7 as select _wstart, _wend, count(*), first(ca), last(ca) from t1 interval(10s); +sql_error create stream streamd71 into streamt71 as select _wstart, _wend, count(*) as ca, first(ca), last(ca) as c2 from t1 interval(10s); + +sql create stream streamd8 into streamt8 as select _wstart, _wend, count(*), first(ca) as c1, last(ca) as c2 from t1 interval(10s); +sql desc streamt8; + +if $rows == 0 then + return -1 +endi + +sql create stream streamd9 into streamt9 as select _wstart, _wend, count(*), first(ca) as c1, last(ca) from t1 interval(10s); +sql desc streamt9; + +if $rows == 0 then + return -1 +endi + +sql_error create stream streamd11 into streamd11 as select _wstart, _wend, count(*), last(ca), last(ca) from t1 interval(10s); + +sql alter local 'keepColumnName' '0' + +sql create stream realtime_meters fill_history 1 into realtime_meters as select last(parttime),first(close),last(close) from meters_test_data partition by tbname state_window(parttime_str); + +sql desc realtime_meters; + +if $rows == 0 then + return -1 +endi + +sql create stream streamd7 into streamt7 as select _wstart, _wend, count(*), first(ca), last(ca) from t1 interval(10s); + +sql desc streamt7; + +if $rows == 0 then + return -1 +endi + +sql create stream streamd71 into streamt71 as select _wstart, _wend, count(*) as ca, first(ca), last(ca) as c2 from t1 interval(10s); + +sql desc streamt71; + +if $rows == 0 then + return -1 +endi + +sql create stream streamd10 into streamd10 as select _wstart, _wend, count(*), first(ca), last(cb) as c2 from t1 interval(10s); + +sql desc streamd10; + +if $rows == 0 then + return -1 +endi + +sql_error create stream streamd11 into streamd11 as select _wstart, _wend, count(*), last(ca), last(ca) from t1 interval(10s); + + +sql create stream streamd12 into streamd12 as select _wstart, _wend, count(*), last(ca), last(cb) as c2 from t1 interval(10s); + +sql desc streamd12; + +if $rows == 0 then + return -1 +endi + sleep 3000 sql drop stream if exists streamd1; diff --git a/tests/system-test/2-query/blockSMA.py b/tests/system-test/2-query/blockSMA.py new file mode 100644 index 0000000000..85c0189e27 --- /dev/null +++ b/tests/system-test/2-query/blockSMA.py @@ -0,0 +1,146 @@ +from wsgiref.headers import tspecials +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10000 + self.ts = 1537146000000 + + def run(self): + dbname = "db" + tdSql.prepare() + + tdSql.execute(f'''create table {dbname}.ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + for i in range(self.rowNum): + tdSql.execute(f"insert into {dbname}.ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i % 127 + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i % 255 + 1, i + 1, i + 1, i + 1)) + + + tdSql.execute('flush database db') + + # test functions using sma result + tdSql.query(f"select count(col1),min(col1),max(col1),avg(col1),sum(col1),spread(col1),percentile(col1, 0),first(col1),last(col1) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 127) + tdSql.checkData(0, 3, 63.8449) + tdSql.checkData(0, 4, 638449) + tdSql.checkData(0, 5, 126.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 94) + + tdSql.query(f"select count(col2),min(col2),max(col2),avg(col2),sum(col2),spread(col2),percentile(col2, 0),first(col2),last(col2) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col3),min(col3),max(col3),avg(col3),sum(col3),spread(col3),percentile(col3, 0),first(col3),last(col3) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col4),min(col4),max(col4),avg(col4),sum(col4),spread(col4),percentile(col4, 0),first(col4),last(col4) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col5),min(col5),max(col5),avg(col5),sum(col5),spread(col5),percentile(col5, 0),first(col5),last(col5) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(0, 2, 9999.09961) + tdSql.checkData(0, 3, 4999.599985846) + tdSql.checkData(0, 4, 49995999.858455874) + tdSql.checkData(0, 5, 9998.999609374) + tdSql.checkData(0, 6, 0.100000001) + tdSql.checkData(0, 7, 0.1) + tdSql.checkData(0, 8, 9999.09961) + + tdSql.query(f"select count(col6),min(col6),max(col6),avg(col6),sum(col6),spread(col6),percentile(col6, 0),first(col6),last(col6) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(0, 2, 9999.100000000) + tdSql.checkData(0, 3, 4999.600000001) + tdSql.checkData(0, 4, 49996000.000005305) + tdSql.checkData(0, 5, 9999.000000000) + tdSql.checkData(0, 6, 0.1) + tdSql.checkData(0, 7, 0.1) + tdSql.checkData(0, 8, 9999.1) + + tdSql.query(f"select count(col11),min(col11),max(col11),avg(col11),sum(col11),spread(col11),percentile(col11, 0),first(col11),last(col11) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 255) + tdSql.checkData(0, 3, 127.45) + tdSql.checkData(0, 4, 1274500) + tdSql.checkData(0, 5, 254.000000000) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 55) + + tdSql.query(f"select count(col12),min(col12),max(col12),avg(col12),sum(col12),spread(col12),percentile(col12, 0),first(col12),last(col12) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col13),min(col13),max(col13),avg(col13),sum(col13),spread(col13),percentile(col13, 0),first(col13),last(col13) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + tdSql.query(f"select count(col14),min(col14),max(col14),avg(col14),sum(col14),spread(col14),percentile(col14, 0),first(col14),last(col14) from {dbname}.ntb") + tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 10000) + tdSql.checkData(0, 3, 5000.5) + tdSql.checkData(0, 4, 50005000) + tdSql.checkData(0, 5, 9999.0) + tdSql.checkData(0, 6, 1.0) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, 10000) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 1c2a6055bc..df460df5c3 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -408,8 +408,8 @@ class TDTestCase: # test group by & order by json tag tdSql.query(f"select ts,jtag->'tag1' from {dbname}.jsons1 partition by jtag->'tag1' order by jtag->'tag1' desc") tdSql.checkRows(11) - tdSql.checkData(0, 1, '"femail"') - tdSql.checkData(2, 1, '"收到货"') + tdSql.checkData(0, 1, '"收到货"') + tdSql.checkData(2, 1, '"femail"') tdSql.checkData(7, 1, "false") @@ -421,9 +421,10 @@ class TDTestCase: tdSql.query(f"select count(*),jtag->'tag1' from {dbname}.jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") tdSql.checkRows(8) tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, '"femail"') + tdSql.checkData(0, 1, '"收到货"') + tdSql.checkData(1, 1, '"femail"') tdSql.checkData(1, 0, 2) - tdSql.checkData(1, 1, '"收到货"') + tdSql.checkData(2, 0, 1) tdSql.checkData(2, 1, "11.000000000") tdSql.checkData(5, 0, 1) @@ -437,7 +438,7 @@ class TDTestCase: tdSql.checkData(5, 0, 1) tdSql.checkData(5, 1, "11.000000000") tdSql.checkData(7, 0, 2) - tdSql.checkData(7, 1, '"femail"') + tdSql.checkData(7, 1, '"收到货"') # test stddev with group by json tag tdSql.query(f"select stddev(dataint),jtag->'tag1' from {dbname}.jsons1 group by jtag->'tag1' order by jtag->'tag1'") @@ -445,8 +446,8 @@ class TDTestCase: tdSql.checkData(0, 1, None) tdSql.checkData(4, 0, 0) tdSql.checkData(4, 1, "5.000000000") - tdSql.checkData(7, 0, 11) - tdSql.checkData(7, 1, '"femail"') + tdSql.checkData(6, 0, 11) + tdSql.checkData(7, 1, '"收到货"') res = tdSql.getColNameList(f"select stddev(dataint),jsons1.jtag->'tag1' from {dbname}.jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") cname_list = [] diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index ea78c90abd..c5ae44214a 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -61,7 +61,7 @@ class TDTestCase: def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): sql = "insert into %s.consumeinfo values "%cdbName - sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + sql += "(now + %ds, %d, '%s', '%s', %d, %d, %d)"%(consumerId, consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) tdLog.info("consume info sql: %s"%sql) tdSql.query(sql) @@ -174,12 +174,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdLog.info("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -191,10 +192,11 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) prepareEnvThread2.start() @@ -259,12 +261,13 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -276,10 +279,11 @@ class TDTestCase: 'ctbNum': 10, \ 'rowsPerTbl': 5000, \ 'batchNum': 100, \ + 'replica': self.replicaVar, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) prepareEnvThread2.start() diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 9bfc01529f..87832ac0ef 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -117,7 +117,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") @@ -186,7 +186,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 975f89cbd7..9bb8c4cc0d 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -53,7 +53,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -116,7 +116,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") @@ -185,7 +185,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index c0c459d315..b910caf420 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -53,7 +53,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -116,7 +116,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) consumerId = 4 @@ -188,7 +188,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py index 3b72b4aea5..e59e3d6ecd 100644 --- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py +++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py @@ -53,7 +53,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar,wal_retention_size=-1, wal_retention_period=-1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -146,7 +146,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) @@ -247,7 +247,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) @@ -335,7 +335,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py index 49a475ff16..4f595788da 100644 --- a/tests/system-test/7-tmq/tmqUpdateWithConsume.py +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -53,7 +53,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1, wal_retention_period=-1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar, wal_retention_size=-1, wal_retention_period=-1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -124,7 +124,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)