diff --git a/.gitignore b/.gitignore index 1bfbf00cd5..b62bd62d9c 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ source/libs/parser/inc/sql.* tests/script/tmqResult.txt +tests/tmqResult.txt # Emacs # -*- mode: gitignore; -*- diff --git a/2.0/src/client/inc/tsclient.h b/2.0/src/client/inc/tsclient.h index e14a3123ad..d5021f663d 100644 --- a/2.0/src/client/inc/tsclient.h +++ b/2.0/src/client/inc/tsclient.h @@ -116,7 +116,7 @@ typedef struct SParsedDataColInfo { uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) uint16_t extendedVarLen; uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - int32_t * boundedColumns; // bound column idx according to schema + int32_t *boundColumns; // bound column idx according to schema SBoundColumn * cols; SBoundIdxInfo *colIdxInfo; int8_t orderStatus; // bound columns @@ -125,7 +125,7 @@ typedef struct SParsedDataColInfo { #define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) typedef struct { - uint8_t memRowType; // default is 0, that is SDataRow + uint8_t rowType; // default is 0, that is SDataRow int32_t rowSize; } SMemRowBuilder; @@ -137,17 +137,17 @@ void destroyMemRowBuilder(SMemRowBuilder *pBuilder); /** * @brief * - * @param memRowType + * @param rowType * @param spd * @param idx the absolute bound index of columns * @return FORCE_INLINE */ -static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd, - int32_t idx, int32_t *toffset, int16_t *colId) { +static FORCE_INLINE void tscGetSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, int32_t idx, + int32_t *toffset, int16_t *colId) { int32_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundedColumns[idx]; - if (isDataRowT(memRowType)) { + schemaIdx = spd->boundColumns[idx]; + if (isDataRowT(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart } else { *toffset = idx * sizeof(SColIdx); // the offset of SColIdx @@ -155,7 +155,7 @@ static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRow } else { ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; - if (isDataRowT(memRowType)) { + if (isDataRowT(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; } else { *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); diff --git a/2.0/src/client/src/tscParseInsert.c b/2.0/src/client/src/tscParseInsert.c index b3f7c076e3..a1f5899cee 100644 --- a/2.0/src/client/src/tscParseInsert.c +++ b/2.0/src/client/src/tscParseInsert.c @@ -428,7 +428,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql - int32_t colIndex = spd->boundedColumns[i]; + int32_t colIndex = spd->boundColumns[i]; char *start = row + spd->cols[colIndex].offset; @@ -495,7 +495,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); int32_t toffset = -1; int16_t colId = -1; - tscGetMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); + tscGetSTSRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); int32_t ret = tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, colId); @@ -630,7 +630,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32 pColInfo->numOfCols = numOfCols; pColInfo->numOfBound = numOfCols; pColInfo->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); + pColInfo->boundColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); pColInfo->colIdxInfo = NULL; pColInfo->flen = 0; @@ -656,7 +656,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32 default: break; } - pColInfo->boundedColumns[i] = i; + pColInfo->boundColumns[i] = i; } pColInfo->allNullLen += pColInfo->flen; pColInfo->boundNullLen = pColInfo->allNullLen; // default set allNullLen @@ -991,7 +991,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } for (int i = 0; i < spd.numOfBound; ++i) { - SSchema* pSchema = &pTagSchema[spd.boundedColumns[i]]; + SSchema *pSchema = &pTagSchema[spd.boundColumns[i]]; index = 0; sToken = tStrGetToken(sql, &index, true); @@ -1158,7 +1158,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->numOfBound = 0; pColInfo->boundNullLen = 0; - memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols); + memset(pColInfo->boundColumns, 0, sizeof(int32_t) * nCols); for (int32_t i = 0; i < nCols; ++i) { pColInfo->cols[i].valStat = VAL_STAT_NONE; } @@ -1205,7 +1205,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundedColumns[pColInfo->numOfBound] = t; + pColInfo->boundColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: @@ -1239,7 +1239,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundedColumns[pColInfo->numOfBound] = t; + pColInfo->boundColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: @@ -1279,7 +1279,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } SBoundIdxInfo *pColIdx = pColInfo->colIdxInfo; for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { - pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundedColumns[i]; + pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundColumns[i]; pColIdx[i].boundIdx = i; } qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); @@ -1289,7 +1289,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - memset(&pColInfo->boundedColumns[pColInfo->numOfBound], 0, + memset(&pColInfo->boundColumns[pColInfo->numOfBound], 0, sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); return TSDB_CODE_SUCCESS; diff --git a/2.0/src/client/src/tscUtil.c b/2.0/src/client/src/tscUtil.c index 3a8c740ea1..449f1ea453 100644 --- a/2.0/src/client/src/tscUtil.c +++ b/2.0/src/client/src/tscUtil.c @@ -1554,7 +1554,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { } void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) { - tfree(pColInfo->boundedColumns); + tfree(pColInfo->boundColumns); tfree(pColInfo->cols); tfree(pColInfo->colIdxInfo); } diff --git a/example/src/tmq.c b/example/src/tmq.c index 35c3e655d6..3b4b6afbaf 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -44,7 +44,7 @@ int32_t init_env() { pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); if (taos_errno(pRes) != 0) { - printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); diff --git a/include/client/taos.h b/include/client/taos.h index 82f0635612..55e1d1c422 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -213,8 +213,10 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, v DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); +DLL_EXPORT void tmq_list_destroy(tmq_list_t *); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); +DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); @@ -244,8 +246,8 @@ enum tmq_conf_res_t { typedef enum tmq_conf_res_t tmq_conf_res_t; DLL_EXPORT tmq_conf_t *tmq_conf_new(); -DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); +DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); // temporary used function for demo only @@ -256,6 +258,7 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message); +DLL_EXPORT char *tmq_get_topic_schema(tmq_t *tmq, const char *topic); /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */ DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index c91efc3ce2..67611d9563 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -54,13 +54,16 @@ typedef struct SColumnDataAgg { } SColumnDataAgg; typedef struct SDataBlockInfo { - STimeWindow window; - int32_t rows; - int32_t rowSize; - int16_t numOfCols; - int16_t hasVarCol; - union {int64_t uid; int64_t blockId;}; - int64_t groupId; // no need to serialize + STimeWindow window; + int32_t rows; + int32_t rowSize; + int16_t numOfCols; + int16_t hasVarCol; + union { + int64_t uid; + int64_t blockId; + }; + int64_t groupId; // no need to serialize } SDataBlockInfo; typedef struct SSDataBlock { @@ -92,7 +95,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); int32_t tEncodeDataBlocks(void** buf, const SArray* blocks); -void* tDecodeDataBlocks(const void* buf, SArray* blocks); +void* tDecodeDataBlocks(const void* buf, SArray** blocks); static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { // WARNING: do not use info.numOfCols, @@ -101,16 +104,16 @@ static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { for (int32_t i = 0; i < numOfOutput; ++i) { SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - tfree(pColInfoData->varmeta.offset); + taosMemoryFreeClear(pColInfoData->varmeta.offset); } else { - tfree(pColInfoData->nullbitmap); + taosMemoryFreeClear(pColInfoData->nullbitmap); } - tfree(pColInfoData->pData); + taosMemoryFreeClear(pColInfoData->pData); } taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockAgg); + taosMemoryFreeClear(pBlock->pBlockAgg); } static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); } @@ -124,7 +127,7 @@ static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics); if (pRsp->numOfTopics == 0) return tlen; - tlen += tEncodeSSchemaWrapper(buf, pRsp->schema); + tlen += taosEncodeSSchemaWrapper(buf, pRsp->schema); if (pRsp->pBlockData) { sz = taosArrayGetSize(pRsp->pBlockData); } @@ -144,9 +147,9 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics); if (pRsp->numOfTopics == 0) return buf; - pRsp->schema = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); + pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper)); if (pRsp->schema == NULL) return NULL; - buf = tDecodeSSchemaWrapper(buf, pRsp->schema); + buf = taosDecodeSSchemaWrapper(buf, pRsp->schema); buf = taosDecodeFixedI32(buf, &sz); pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); for (int32_t i = 0; i < sz; i++) { @@ -160,9 +163,9 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { if (pRsp->schema) { if (pRsp->schema->nCols) { - tfree(pRsp->schema->pSchema); + taosMemoryFreeClear(pRsp->schema->pSchema); } - free(pRsp->schema); + taosMemoryFree(pRsp->schema); } taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner); pRsp->pBlockData = NULL; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index f181c26e92..20d743046f 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -102,7 +102,8 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2); +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, + uint32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); @@ -113,7 +114,8 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock); size_t blockDataGetNumOfRows(const SSDataBlock* pBlock); int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); -int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize); +int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, + int32_t pageSize); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf); @@ -136,6 +138,8 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); void* blockDataDestroy(SSDataBlock* pBlock); +void blockDebugShowData(const SArray* dataBlocks); + #ifdef __cplusplus } #endif diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 89a0e6b4dc..a2899ead8e 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -59,12 +59,15 @@ extern "C" { } while (0); // ----------------- TSDB COLUMN DEFINITION +#pragma pack(push, 1) typedef struct { - int8_t type; // Column type - col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) - int16_t bytes; // column bytes (restore to int16_t in case of misuse) - uint16_t offset; // point offset in STpRow after the header part. + col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) + int32_t type : 8; // column type + int32_t bytes : 24; // column bytes (restore to int32_t in case of misuse) + int32_t sma : 8; // block SMA: 0, no SMA, 1, sum/min/max, 2, ... + int32_t offset : 24; // point offset in STpRow after the header part. } STColumn; +#pragma pack(pop) #define colType(col) ((col)->type) #define colColId(col) ((col)->colId) @@ -93,7 +96,7 @@ typedef struct { #define schemaFLen(s) ((s)->flen) #define schemaVLen(s) ((s)->vlen) #define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) tfree((s)) +#define tdFreeSchema(s) taosMemoryFreeClear((s)) STSchema *tdDupSchema(const STSchema *pSchema); int32_t tdEncodeSchema(void **buf, STSchema *pSchema); @@ -136,7 +139,7 @@ typedef struct { int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition @@ -493,7 +496,7 @@ typedef struct { #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -#define kvRowFree(r) tfree(r) +#define kvRowFree(r) taosMemoryFreeClear(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) #define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) #define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) @@ -590,10 +593,10 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { +static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; - SColIdx *pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); + SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); if (pColIdx == NULL) return -1; pBuilder->pColIdx = pColIdx; } @@ -608,7 +611,7 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t col while (tlen > pBuilder->alloc - pBuilder->size) { pBuilder->alloc *= 2; } - void *buf = realloc(pBuilder->buf, pBuilder->alloc); + void *buf = taosMemoryRealloc(pBuilder->buf, pBuilder->alloc); if (buf == NULL) return -1; pBuilder->buf = buf; } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6181c49844..bdb6181884 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -201,26 +201,18 @@ typedef struct SEp { typedef struct { int32_t contLen; - union { - int32_t vgId; - int32_t streamTaskId; - }; + int32_t vgId; } SMsgHead; -typedef struct { - int32_t workerType; - int32_t streamTaskId; -} SStreamExecMsgHead; - // Submit message for one table typedef struct SSubmitBlk { int64_t uid; // table unique id - int64_t suid; // stable id - int32_t padding; // TODO just for padding here + int64_t suid; // stable id int32_t sversion; // data schema version int32_t dataLen; // data part length, not including the SSubmitBlk head int32_t schemaLen; // schema length, if length is 0, no schema exists int16_t numOfRows; // total number of rows in current submit block + int16_t padding; // TODO just for padding here char data[]; } SSubmitBlk; @@ -267,10 +259,10 @@ typedef struct { } SSubmitRsp; typedef struct SSchema { - int8_t type; - int32_t colId; - int32_t bytes; - char name[TSDB_COL_NAME_LEN]; + int8_t type; + col_id_t colId; + int32_t bytes; + char name[TSDB_COL_NAME_LEN]; } SSchema; typedef struct { @@ -446,8 +438,8 @@ typedef struct { */ typedef struct { union { - int16_t colId; - int16_t slotId; + col_id_t colId; + int16_t slotId; }; int16_t type; @@ -477,7 +469,8 @@ typedef struct { int32_t tz; // query client timezone char intervalUnit; char slidingUnit; - char offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. + char + offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration. int8_t precision; int64_t interval; int64_t sliding; @@ -1132,7 +1125,6 @@ int32_t tDeserializeSSchedulerHbReq(void* buf, int32_t bufLen, SSchedulerHbReq* void tFreeSSchedulerHbReq(SSchedulerHbReq* pReq); typedef struct { - uint64_t seqId; SQueryNodeEpId epId; SArray* taskStatus; // SArray } SSchedulerHbRsp; @@ -1316,7 +1308,7 @@ typedef struct { } SMqRebSubscribe; static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { - SMqRebSubscribe* pRebSub = (SMqRebSubscribe*)calloc(1, sizeof(SMqRebSubscribe)); + SMqRebSubscribe* pRebSub = (SMqRebSubscribe*)taosMemoryCalloc(1, sizeof(SMqRebSubscribe)); if (pRebSub == NULL) { goto _err; } @@ -1338,7 +1330,7 @@ _err: taosArrayDestroy(pRebSub->lostConsumers); taosArrayDestroy(pRebSub->removedConsumers); taosArrayDestroy(pRebSub->newConsumers); - tfree(pRebSub); + taosMemoryFreeClear(pRebSub); return NULL; } @@ -1639,7 +1631,7 @@ static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { void* pIter = taosHashIterate(info, NULL); while (pIter != NULL) { SKv* kv = (SKv*)pIter; - tfree(kv->value); + taosMemoryFreeClear(kv->value); pIter = taosHashIterate(info, pIter); } } @@ -1662,13 +1654,13 @@ static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { } else { taosArrayDestroy(req->reqs); } - free(pReq); + taosMemoryFree(pReq); } static FORCE_INLINE void tFreeClientKv(void* pKv) { SKv* kv = (SKv*)pKv; if (kv) { - tfree(kv->value); + taosMemoryFreeClear(kv->value); } } @@ -1695,7 +1687,7 @@ static FORCE_INLINE int32_t tEncodeSKv(SCoder* pEncoder, const SKv* pKv) { static FORCE_INLINE int32_t tDecodeSKv(SCoder* pDecoder, SKv* pKv) { if (tDecodeI32(pDecoder, &pKv->key) < 0) return -1; if (tDecodeI32(pDecoder, &pKv->valueLen) < 0) return -1; - pKv->value = malloc(pKv->valueLen + 1); + pKv->value = taosMemoryMalloc(pKv->valueLen + 1); if (pKv->value == NULL) return -1; if (tDecodeCStrTo(pDecoder, (char*)pKv->value) < 0) return -1; return 0; @@ -1909,7 +1901,7 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSchema->type); tlen += taosEncodeFixedI32(buf, pSchema->bytes); - tlen += taosEncodeFixedI32(buf, pSchema->colId); + tlen += taosEncodeFixedI16(buf, pSchema->colId); tlen += taosEncodeString(buf, pSchema->name); return tlen; } @@ -1917,7 +1909,7 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { buf = taosDecodeFixedI8(buf, &pSchema->type); buf = taosDecodeFixedI32(buf, &pSchema->bytes); - buf = taosDecodeFixedI32(buf, &pSchema->colId); + buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeStringTo(buf, pSchema->name); return buf; } @@ -1925,7 +1917,7 @@ static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) { if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1; - if (tEncodeI32(pEncoder, pSchema->colId) < 0) return -1; + if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1; if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; return 0; } @@ -1933,12 +1925,12 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) { if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1; - if (tDecodeI32(pDecoder, &pSchema->colId) < 0) return -1; + if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1; if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; return 0; } -static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { +static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { int32_t tlen = 0; tlen += taosEncodeFixedU32(buf, pSW->nCols); for (int32_t i = 0; i < pSW->nCols; i++) { @@ -1947,9 +1939,9 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapp return tlen; } -static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { +static FORCE_INLINE void* taosDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { buf = taosDecodeFixedU32(buf, &pSW->nCols); - pSW->pSchema = (SSchema*)calloc(pSW->nCols, sizeof(SSchema)); + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { return NULL; } @@ -1960,6 +1952,27 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) return buf; } +static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) { + if (tEncodeU32(pEncoder, pSW->nCols) < 0) return -1; + for (int32_t i = 0; i < pSW->nCols; i++) { + if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; + } + return pEncoder->pos; +} + +static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapper* pSW) { + if (tDecodeU32(pDecoder, &pSW->nCols) < 0) return -1; + void* ptr = taosMemoryRealloc(pSW->pSchema, pSW->nCols * sizeof(SSchema)); + if (ptr == NULL) { + return -1; + } + pSW->pSchema = (SSchema*)ptr; + for (int32_t i = 0; i < pSW->nCols; i++) { + if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; + } + return 0; +} + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; @@ -2084,8 +2097,8 @@ typedef struct { static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { if (pSma) { - tfree(pSma->expr); - tfree(pSma->tagsFilter); + taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); } } @@ -2095,14 +2108,14 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) { for (uint32_t i = 0; i < pSW->number; ++i) { tdDestroyTSma(pSW->tSma + i); } - tfree(pSW->tSma); + taosMemoryFreeClear(pSW->tSma); } } } static FORCE_INLINE void tdFreeTSmaWrapper(STSmaWrapper* pSW) { tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); } static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { @@ -2180,7 +2193,7 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { buf = taosDecodeFixedU32(buf, &pSW->number); - pSW->tSma = (STSma*)calloc(pSW->number, sizeof(STSma)); + pSW->tSma = (STSma*)taosMemoryCalloc(pSW->number, sizeof(STSma)); if (pSW->tSma == NULL) { return NULL; } @@ -2190,7 +2203,7 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { for (uint32_t j = i; j >= 0; --i) { tdDestroyTSma(pSW->tSma + j); } - free(pSW->tSma); + taosMemoryFree(pSW->tSma); return NULL; } } @@ -2365,108 +2378,6 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } return buf; } - -enum { - STREAM_TASK_STATUS__RUNNING = 1, - STREAM_TASK_STATUS__STOP, -}; - -enum { - STREAM_NEXT_OP_DST__VND = 1, - STREAM_NEXT_OP_DST__SND, -}; - -enum { - STREAM_SOURCE_TYPE__NONE = 1, - STREAM_SOURCE_TYPE__SUPER, - STREAM_SOURCE_TYPE__CHILD, - STREAM_SOURCE_TYPE__NORMAL, -}; - -enum { - STREAM_SINK_TYPE__NONE = 1, - STREAM_SINK_TYPE__INPLACE, - STREAM_SINK_TYPE__ASSIGNED, - STREAM_SINK_TYPE__MULTIPLE, - STREAM_SINK_TYPE__TEMPORARY, -}; - -enum { - STREAM_TYPE__NORMAL = 1, - STREAM_TYPE__SMA, -}; - -typedef struct { - void* inputHandle; - void* executor; -} SStreamRunner; - -typedef struct { - int64_t streamId; - int32_t taskId; - int32_t level; - int8_t status; - int8_t parallelizable; - - // vnode or snode - int8_t nextOpDst; - - int8_t sourceType; - int8_t sinkType; - - // for sink type assigned - int32_t sinkVgId; - SEpSet NextOpEp; - - // executor meta info - char* qmsg; - - // followings are not applied to encoder and decoder - int8_t numOfRunners; - SStreamRunner runner[8]; -} SStreamTask; - -static FORCE_INLINE SStreamTask* streamTaskNew(int64_t streamId) { - SStreamTask* pTask = (SStreamTask*)calloc(1, sizeof(SStreamTask)); - if (pTask == NULL) { - return NULL; - } - pTask->taskId = tGenIdPI32(); - pTask->streamId = streamId; - pTask->status = STREAM_TASK_STATUS__RUNNING; - pTask->qmsg = NULL; - return pTask; -} - -int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask); -int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask); -void tFreeSStreamTask(SStreamTask* pTask); - -typedef struct { - SMsgHead head; - SStreamTask* task; -} SStreamTaskDeployReq; - -typedef struct { - int32_t reserved; -} SStreamTaskDeployRsp; - -typedef struct { - SStreamExecMsgHead head; - SArray* data; // SArray -} SStreamTaskExecReq; - -typedef struct { - int32_t reserved; -} SStreamTaskExecRsp; - -typedef struct { - SMsgHead head; - int64_t streamId; - int64_t version; - SArray* res; // SArray -} SStreamSinkReq; - #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 54a145ff33..1b3311b400 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -25,7 +25,16 @@ extern "C" { typedef struct SRpcMsg SRpcMsg; typedef struct SEpSet SEpSet; typedef struct SMgmtWrapper SMgmtWrapper; -typedef enum { QUERY_QUEUE, FETCH_QUEUE, WRITE_QUEUE, APPLY_QUEUE, SYNC_QUEUE, QUEUE_MAX } EQueueType; +typedef enum { + QUERY_QUEUE, + FETCH_QUEUE, + READ_QUEUE, + WRITE_QUEUE, + APPLY_QUEUE, + SYNC_QUEUE, + MERGE_QUEUE, + QUEUE_MAX, +} EQueueType; typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index f9ce69925b..051ee34644 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -193,6 +193,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_PIPE_EXEC, "vnode-task-pipe-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_MERGE_EXEC, "vnode-task-merge-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_WRITE_EXEC, "vnode-task-write-exec", SStreamTaskExecReq, SStreamTaskExecRsp) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) @@ -206,7 +209,13 @@ enum { TD_NEW_MSG_SEG(TDMT_SND_MSG) TD_DEF_MSG_TYPE(TDMT_SND_TASK_DEPLOY, "snode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_SND_TASK_EXEC, "snode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_SND_TASK_PIPE_EXEC, "snode-task-pipe-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + TD_DEF_MSG_TYPE(TDMT_SND_TASK_MERGE_EXEC, "snode-task-merge-exec", SStreamTaskExecReq, SStreamTaskExecRsp) + // Requests handled by SCHEDULER + TD_NEW_MSG_SEG(TDMT_SCH_MSG) + TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "scheduler-link-broken", NULL, NULL) + #if defined(TD_MSG_NUMBER_) TDMT_MAX #endif diff --git a/include/common/trow.h b/include/common/trow.h index fc99cbc5b2..df28bc9962 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -671,8 +671,9 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa * @param colIdx sorted column index, start from 0 * @return FORCE_INLINE */ -static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, int16_t colId, int8_t colType, TDRowValT valType, - const void *val, bool isCopyVarData, int32_t offset, int16_t colIdx) { +static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, + TDRowValT valType, const void *val, bool isCopyVarData, int32_t offset, + col_id_t colIdx) { STSRow *pRow = pBuilder->pBuf; if (!val) { #ifdef TD_SUPPORT_BITMAP diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 59af14c226..87dc752703 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -29,6 +29,7 @@ typedef uint32_t TDRowLenT; typedef uint8_t TDRowValT; typedef int16_t col_id_t; typedef int8_t col_type_t; +typedef int32_t col_bytes_t; #pragma pack(push, 1) typedef struct { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index dd5f7fc104..9f0d4b11c2 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -112,7 +112,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t d int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId); -int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName); +int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName); int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 35832fa298..85a9cd0b23 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -57,18 +57,19 @@ typedef enum EFunctionType { // math function FUNCTION_TYPE_ABS = 1000, - FUNCTION_TYPE_ACOS, - FUNCTION_TYPE_ASION, - FUNCTION_TYPE_ATAN, - FUNCTION_TYPE_CEIL, - FUNCTION_TYPE_COS, - FUNCTION_TYPE_FLOOR, FUNCTION_TYPE_LOG, FUNCTION_TYPE_POW, - FUNCTION_TYPE_ROUND, - FUNCTION_TYPE_SIN, FUNCTION_TYPE_SQRT, + FUNCTION_TYPE_CEIL, + FUNCTION_TYPE_FLOOR, + FUNCTION_TYPE_ROUND, + + FUNCTION_TYPE_SIN, + FUNCTION_TYPE_COS, FUNCTION_TYPE_TAN, + FUNCTION_TYPE_ASIN, + FUNCTION_TYPE_ACOS, + FUNCTION_TYPE_ATAN, // string function FUNCTION_TYPE_CHAR_LENGTH = 1500, diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 4c83a30bb9..9b8739a4f3 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -120,6 +120,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, QUERY_NODE_LOGIC_PLAN_EXCHANGE, QUERY_NODE_LOGIC_PLAN_WINDOW, + QUERY_NODE_LOGIC_PLAN_SORT, QUERY_NODE_LOGIC_SUBPLAN, QUERY_NODE_LOGIC_PLAN, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 794e0ca85a..08792b6f8f 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -48,6 +48,7 @@ typedef struct SScanLogicNode { uint8_t scanFlag; // denotes reversed scan of data or not STimeWindow scanRange; SName tableName; + bool showRewrite; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -65,6 +66,7 @@ typedef struct SAggLogicNode { typedef struct SProjectLogicNode { SLogicNode node; SNodeList* pProjections; + char stmtName[TSDB_TABLE_NAME_LEN]; } SProjectLogicNode; typedef struct SVnodeModifLogicNode { @@ -96,8 +98,14 @@ typedef struct SWindowLogicNode { int8_t slidingUnit; SFillNode* pFill; int64_t sessionGap; + SNode* pTspk; } SWindowLogicNode; +typedef struct SSortLogicNode { + SLogicNode node; + SNodeList* pSortKeys; +} SSortLogicNode; + typedef enum ESubplanType { SUBPLAN_TYPE_MERGE = 1, SUBPLAN_TYPE_PARTIAL, @@ -171,6 +179,8 @@ typedef SScanPhysiNode SStreamScanPhysiNode; typedef struct SSystemTableScanPhysiNode { SScanPhysiNode scan; SEpSet mgmtEpSet; + bool showRewrite; + int32_t accountId; } SSystemTableScanPhysiNode; typedef struct STableScanPhysiNode { @@ -197,7 +207,7 @@ typedef struct SJoinPhysiNode { typedef struct SAggPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function - SNodeList* pGroupKeys; // SColumnRefNode list + SNodeList* pGroupKeys; SNodeList* pAggFuncs; } SAggPhysiNode; @@ -222,6 +232,7 @@ typedef struct SWinodwPhysiNode { typedef struct SIntervalPhysiNode { SWinodwPhysiNode window; + SNode* pTspk; // timestamp primary key int64_t interval; int64_t offset; int64_t sliding; @@ -235,6 +246,12 @@ typedef struct SSessionWinodwPhysiNode { int64_t gap; } SSessionWinodwPhysiNode; +typedef struct SSortPhysiNode { + SPhysiNode node; + SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function + SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode +} SSortPhysiNode; + typedef struct SDataSinkNode { ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 616e24d67d..66f60bde98 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -56,7 +56,7 @@ typedef enum EColumnType { typedef struct SColumnNode { SExprNode node; // QUERY_NODE_COLUMN uint64_t tableId; - int16_t colId; + col_id_t colId; EColumnType colType; // column or tag char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; @@ -191,12 +191,13 @@ typedef struct SStateWindowNode { typedef struct SSessionWindowNode { ENodeType type; // QUERY_NODE_SESSION_WINDOW - SNode* pCol; + SNode* pCol; // timestamp primary key SNode* pGap; // gap between two session window(in microseconds) } SSessionWindowNode; typedef struct SIntervalWindowNode { ENodeType type; // QUERY_NODE_INTERVAL_WINDOW + SNode* pCol; // timestamp primary key SNode* pInterval; // SValueNode SNode* pOffset; // SValueNode SNode* pSliding; // SValueNode @@ -231,6 +232,7 @@ typedef struct SSelectStmt { SNodeList* pOrderByList; // SOrderByExprNode SNode* pLimit; SNode* pSlimit; + char stmtName[TSDB_TABLE_NAME_LEN]; } SSelectStmt; typedef enum ESetOperatorType { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 2254298e5c..0747534721 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -54,6 +54,7 @@ typedef struct SQuery { int32_t msgType; SArray* pDbList; SArray* pTableList; + bool showRewrite; } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 38b30ec01e..8db78fccf5 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -29,6 +29,7 @@ typedef struct SPlanContext { SNode* pAstRoot; bool topicQuery; bool streamQuery; + bool showRewrite; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 839feb7800..f487e5ae22 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -43,10 +43,10 @@ typedef enum { } ETaskType; typedef struct STableComInfo { - uint8_t numOfTags; // the number of tags in schema - uint8_t precision; // the number of precision - int16_t numOfColumns; // the number of columns - int32_t rowSize; // row size of the schema + uint8_t numOfTags; // the number of tags in schema + uint8_t precision; // the number of precision + col_id_t numOfColumns; // the number of columns + int32_t rowSize; // row size of the schema } STableComInfo; typedef struct SIndexMeta { @@ -150,6 +150,8 @@ int32_t cleanupTaskQueue(); */ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); +int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *ctx); + /** * Asynchronously send message to server, after the response received, the callback will be incured. * @@ -171,7 +173,7 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); -SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); +SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 944ac97ddb..0846841cef 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -28,6 +28,7 @@ enum { NODE_TYPE_VNODE = 1, NODE_TYPE_QNODE, NODE_TYPE_SNODE, + NODE_TYPE_MNODE, }; diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index e55c0b9e76..12c03afcb6 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -42,6 +42,21 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut); +int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t sqrtFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + +int32_t sinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t cosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t tanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t asinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t acosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + +int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h new file mode 100644 index 0000000000..177fe39397 --- /dev/null +++ b/include/libs/stream/tstream.h @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdatablock.h" +#include "tmsg.h" +#include "tmsgcb.h" +#include "trpc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _TSTREAM_H_ +#define _TSTREAM_H_ + +enum { + STREAM_TASK_STATUS__RUNNING = 1, + STREAM_TASK_STATUS__STOP, +}; + +#if 0 +// pipe -> fetch/pipe queue +// merge -> merge queue +// write -> write queue +enum { + TASK_DISPATCH_MSG__SND_PIPE = 1, + TASK_DISPATCH_MSG__SND_MERGE, + TASK_DISPATCH_MSG__VND_PIPE, + TASK_DISPATCH_MSG__VND_MERGE, + TASK_DISPATCH_MSG__VND_WRITE, +}; +#endif + +typedef struct { + int32_t nodeId; // 0 for snode + SEpSet epSet; +} SStreamTaskEp; + +typedef struct { + void* inputHandle; + void* executor; +} SStreamRunner; + +typedef struct { + int8_t parallelizable; + char* qmsg; + // followings are not applicable to encoder and decoder + int8_t numOfRunners; + SStreamRunner* runners; +} STaskExec; + +typedef struct { + int8_t reserved; +} STaskDispatcherInplace; + +typedef struct { + int32_t nodeId; + SEpSet epSet; +} STaskDispatcherFixedEp; + +typedef struct { + int8_t hashMethod; + SArray* info; +} STaskDispatcherShuffle; + +typedef struct { + int8_t reserved; + // not applicable to encoder and decoder + SHashObj* pHash; // groupId to tbuid +} STaskSinkTb; + +typedef struct { + int8_t reserved; +} STaskSinkSma; + +typedef struct { + int8_t reserved; +} STaskSinkFetch; + +typedef struct { + int8_t reserved; +} STaskSinkShow; + +enum { + TASK_SOURCE__SCAN = 1, + TASK_SOURCE__PIPE, + TASK_SOURCE__MERGE, +}; + +enum { + TASK_EXEC__NONE = 1, + TASK_EXEC__PIPE, + TASK_EXEC__MERGE, +}; + +enum { + TASK_DISPATCH__NONE = 1, + TASK_DISPATCH__INPLACE, + TASK_DISPATCH__FIXED, + TASK_DISPATCH__SHUFFLE, +}; + +enum { + TASK_SINK__NONE = 1, + TASK_SINK__TABLE, + TASK_SINK__SMA, + TASK_SINK__FETCH, + TASK_SINK__SHOW, +}; + +typedef struct { + int64_t streamId; + int32_t taskId; + int8_t status; + + int8_t sourceType; + int8_t execType; + int8_t sinkType; + int8_t dispatchType; + int16_t dispatchMsgType; + int32_t downstreamTaskId; + + int32_t nodeId; + SEpSet epSet; + + // source preprocess + + // exec + STaskExec exec; + + // local sink + union { + STaskSinkTb tbSink; + STaskSinkSma smaSink; + STaskSinkFetch fetchSink; + STaskSinkShow showSink; + }; + + // dispatch + union { + STaskDispatcherInplace inplaceDispatcher; + STaskDispatcherFixedEp fixedEpDispatcher; + STaskDispatcherShuffle shuffleDispatcher; + }; + + // state storage + +} SStreamTask; + +SStreamTask* tNewSStreamTask(int64_t streamId); +int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask); +int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask); +void tFreeSStreamTask(SStreamTask* pTask); + +typedef struct { + // SMsgHead head; + SStreamTask* task; +} SStreamTaskDeployReq; + +typedef struct { + int32_t reserved; +} SStreamTaskDeployRsp; + +typedef struct { + // SMsgHead head; + int64_t streamId; + int32_t taskId; + SArray* data; // SArray +} SStreamTaskExecReq; + +int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq); +void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq); +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq); + +typedef struct { + int32_t reserved; +} SStreamTaskExecRsp; + +typedef struct { + // SMsgHead head; + int64_t streamId; + int64_t version; + SArray* res; // SArray +} SStreamSinkReq; + +int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId); + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef _TSTREAM_H_ */ diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 02029a996c..8125de7647 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -52,8 +52,8 @@ typedef struct { char user[TSDB_USER_LEN]; SRpcMsg rpcMsg; int32_t rspLen; - void *pRsp; - void *pNode; + void * pRsp; + void * pNode; } SNodeMsg; typedef struct SRpcInit { @@ -81,13 +81,21 @@ typedef struct SRpcInit { } SRpcInit; typedef struct { - void * val; - int32_t len; - void (*free)(void *arg); + void *val; + int32_t (*clone)(void *src, void **dst); + void (*freeFunc)(const void *arg); } SRpcCtxVal; typedef struct { - SHashObj *args; + int32_t msgType; + void *val; + int32_t (*clone)(void *src, void **dst); + void (*freeFunc)(const void *arg); +} SRpcBrokenlinkVal; + +typedef struct { + SHashObj * args; + SRpcBrokenlinkVal brokenVal; } SRpcCtx; int32_t rpcInit(); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 7342af4d18..8b1ac3ed8d 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -61,30 +61,45 @@ extern "C" { } \ } -#define WAL_HEAD_VER 0 +#define WAL_HEAD_VER 0 #define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) -#define WAL_LOG_SUFFIX "log" +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) +#define WAL_LOG_SUFFIX "log" #define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -#define WAL_MAGIC 0xFAFBFCFDULL +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) +#define WAL_MAGIC 0xFAFBFCFDULL #define WAL_CUR_FAILED 1 #pragma pack(push, 1) -typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType; +typedef enum { + TAOS_WAL_NOLOG = 0, + TAOS_WAL_WRITE = 1, + TAOS_WAL_FSYNC = 2, +} EWalType; + +// used by sync module +typedef struct { + int8_t isWeek; + uint64_t seqNum; + uint64_t term; +} SSyncLogMeta; typedef struct SWalReadHead { int8_t headVer; - int16_t msgType; int8_t reserved; + int16_t msgType; int32_t len; int64_t ingestTs; // not implemented int64_t version; - char body[]; + + // sync meta + SSyncLogMeta syncMeta; + + char body[]; } SWalReadHead; typedef struct { @@ -117,16 +132,16 @@ typedef struct SWal { SWalCfg cfg; int32_t fsyncSeq; // meta - SWalVer vers; + SWalVer vers; TdFilePtr pWriteLogTFile; TdFilePtr pWriteIdxTFile; - int32_t writeCur; - SArray *fileInfoSet; + int32_t writeCur; + SArray *fileInfoSet; // status int64_t totSize; int64_t lastRollSeq; // ctl - int64_t refId; + int64_t refId; TdThreadMutex mutex; // path char path[WAL_PATH_LEN]; @@ -158,6 +173,8 @@ int32_t walAlter(SWal *, SWalCfg *pCfg); void walClose(SWal *); // write +int64_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body, + int32_t bodyLen); int64_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen); void walFsync(SWal *, bool force); diff --git a/include/os/os.h b/include/os/os.h index b05bfab6d0..e7ce7d09ea 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -75,7 +75,6 @@ extern "C" { #include "osDef.h" #include "osDir.h" #include "osEndian.h" -#include "osEnv.h" #include "osFile.h" #include "osLocale.h" #include "osLz4.h" @@ -93,8 +92,9 @@ extern "C" { #include "osTime.h" #include "osTimer.h" #include "osTimezone.h" +#include "osEnv.h" -void osInit(); +void osDefaultInit(); #ifdef __cplusplus } diff --git a/include/os/osEnv.h b/include/os/osEnv.h index ebf4c360dd..14d50858b7 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -22,17 +22,18 @@ extern "C" { #endif -extern char tsOsName[]; -extern char tsTimezone[]; -extern char tsCharset[]; -extern char tsLocale[]; -extern int8_t tsDaylight; -extern bool tsEnableCoreFile; -extern int64_t tsPageSizeKB; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern float tsNumOfCores; -extern int64_t tsTotalMemoryKB; +extern char tsOsName[]; +extern char tsTimezoneStr[]; +extern enum TdTimezone tsTimezone; +extern char tsCharset[]; +extern char tsLocale[]; +extern int8_t tsDaylight; +extern bool tsEnableCoreFile; +extern int64_t tsPageSizeKB; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern float tsNumOfCores; +extern int64_t tsTotalMemoryKB; extern char configDir[]; extern char tsDataDir[]; @@ -43,11 +44,12 @@ extern SDiskSpace tsDataSpace; extern SDiskSpace tsLogSpace; extern SDiskSpace tsTempSpace; -void osInit(); +void osDefaultInit(); void osUpdate(); void osCleanup(); bool osLogSpaceAvailable(); void osSetTimezone(const char *timezone); +void osSetSystemLocale(const char *inLocale, const char *inCharSet); #ifdef __cplusplus } diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 6100419035..62ac82782c 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -20,12 +20,25 @@ extern "C" { #endif -#define tfree(x) \ - do { \ - if (x) { \ - free((void *)(x)); \ - (x) = 0; \ - } \ +// If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following sectio +#ifndef ALLOW_FORBID_FUNC + #define malloc MALLOC_FUNC_TAOS_FORBID + #define calloc CALLOC_FUNC_TAOS_FORBID + #define realloc REALLOC_FUNC_TAOS_FORBID + #define free FREE_FUNC_TAOS_FORBID +#endif + +void *taosMemoryMalloc(int32_t size); +void *taosMemoryCalloc(int32_t num, int32_t size); +void *taosMemoryRealloc(void *ptr, int32_t size); +void taosMemoryFree(const void *ptr); +int32_t taosMemorySize(void *ptr); + +#define taosMemoryFreeClear(ptr) \ + do { \ + taosMemoryFree(ptr); \ + (ptr)=NULL; \ } while (0) #ifdef __cplusplus diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index c8df8c3f3d..3676c4b634 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -26,8 +26,37 @@ extern "C" { #define tzset TZSET_FUNC_TAOS_FORBID #endif -void taosGetSystemTimezone(char *outTimezone); -void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); +enum TdTimezone +{ + TdWestZone12=-12, + TdWestZone11, + TdWestZone10, + TdWestZone9, + TdWestZone8, + TdWestZone7, + TdWestZone6, + TdWestZone5, + TdWestZone4, + TdWestZone3, + TdWestZone2, + TdWestZone1, + TdZeroZone, + TdEastZone1, + TdEastZone2, + TdEastZone3, + TdEastZone4, + TdEastZone5, + TdEastZone6, + TdEastZone7, + TdEastZone8, + TdEastZone9, + TdEastZone10, + TdEastZone11, + TdEastZone12 +}; + +void taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone); +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 87781b6313..994ad7afc6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -16,6 +16,8 @@ #ifndef _TD_UTIL_TAOS_ERROR_H_ #define _TD_UTIL_TAOS_ERROR_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 943a1f9eca..838b175a28 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -351,7 +351,7 @@ static FORCE_INLINE void *taosDecodeString(const void *buf, char **value) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); - *value = (char *)malloc((size_t)size + 1); + *value = (char *)taosMemoryMalloc((size_t)size + 1); if (*value == NULL) return NULL; memcpy(*value, buf, (size_t)size); @@ -386,7 +386,7 @@ static FORCE_INLINE int32_t taosEncodeBinary(void **buf, const void *value, int3 } static FORCE_INLINE void *taosDecodeBinary(const void *buf, void **value, int32_t valueLen) { - *value = malloc((size_t)valueLen); + *value = taosMemoryMalloc((size_t)valueLen); if (*value == NULL) return NULL; memcpy(*value, buf, (size_t)valueLen); diff --git a/include/util/tdef.h b/include/util/tdef.h index fd8194e63e..6c5208ec00 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -109,6 +109,8 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_INS_TABLE_USER_USERS "user_users" #define TSDB_INS_TABLE_VGROUPS "vgroups" +#define TSDB_INS_USER_STABLES_DBNAME_COLID 2 + #define TSDB_TICK_PER_SECOND(precision) \ ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1e3L \ : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) @@ -207,7 +209,7 @@ typedef enum ELogicConditionType { #define TSDB_FUNC_TYPE_AGGREGATE 2 #define TSDB_FUNC_MAX_RETRIEVE 1024 -#define TSDB_INDEX_NAME_LEN 33 // 32 + 1 '\0' +#define TSDB_INDEX_NAME_LEN 65 // 64 + 1 '\0' #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN diff --git a/include/util/tencode.h b/include/util/tencode.h index 7e1b624dfb..cdde378b69 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -406,7 +406,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SCoder* pDecoder, void** val, uin if (tDecodeU64v(pDecoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; - *val = malloc(*len); + *val = taosMemoryMalloc(*len); if (*val == NULL) return -1; memcpy(*val, TD_CODER_CURRENT(pDecoder), *len); diff --git a/include/util/tfreelist.h b/include/util/tfreelist.h index 0a507aeec9..e9b5ca5fca 100644 --- a/include/util/tfreelist.h +++ b/include/util/tfreelist.h @@ -31,7 +31,7 @@ typedef TD_SLIST(SFreeListNode) SFreeList; #define TFL_MALLOC(PTR, TYPE, SIZE, LIST) \ do { \ - void *ptr = malloc((SIZE) + sizeof(struct SFreeListNode)); \ + void *ptr = taosMemoryMalloc((SIZE) + sizeof(struct SFreeListNode)); \ if (ptr) { \ TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \ ptr = ((struct SFreeListNode *)ptr)->payload; \ @@ -49,7 +49,7 @@ static FORCE_INLINE void tFreeListClear(SFreeList *pFL) { pNode = TD_SLIST_HEAD(pFL); if (pNode == NULL) break; TD_SLIST_POP(pFL); - free(pNode); + taosMemoryFree(pNode); } } diff --git a/include/util/tlist.h b/include/util/tlist.h index caa6424918..43833d7ecd 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -216,7 +216,7 @@ typedef struct { #define listNEles(l) TD_DLIST_NELES(l) #define listEleSize(l) ((l)->eleSize) #define isListEmpty(l) (TD_DLIST_NELES(l) == 0) -#define listNodeFree(n) free(n) +#define listNodeFree(n) taosMemoryFree(n) void tdListInit(SList *list, int32_t eleSize); void tdListEmpty(SList *list); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 505c0eeb67..8e67703ce5 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -115,11 +115,11 @@ void destroyTscObj(void *pObj) { atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); taosThreadMutexDestroy(&pTscObj->mutex); - tfree(pTscObj); + taosMemoryFreeClear(pTscObj); } void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) { - STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); + STscObj *pObj = (STscObj *)taosMemoryCalloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -143,7 +143,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); - SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); + SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); if (NULL == pRequest) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -156,7 +156,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty pRequest->type = type; pRequest->pTscObj = pObj; pRequest->body.fp = fp; // not used it yet - pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); @@ -164,17 +164,17 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty } static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { - tfree(pResInfo->pRspMsg); - tfree(pResInfo->length); - tfree(pResInfo->row); - tfree(pResInfo->pCol); - tfree(pResInfo->fields); + taosMemoryFreeClear(pResInfo->pRspMsg); + taosMemoryFreeClear(pResInfo->length); + taosMemoryFreeClear(pResInfo->row); + taosMemoryFreeClear(pResInfo->pCol); + taosMemoryFreeClear(pResInfo->fields); if (pResInfo->convertBuf != NULL) { for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { - tfree(pResInfo->convertBuf[i]); + taosMemoryFreeClear(pResInfo->convertBuf[i]); } - tfree(pResInfo->convertBuf); + taosMemoryFreeClear(pResInfo->convertBuf); } } @@ -184,20 +184,24 @@ static void doDestroyRequest(void *p) { assert(RID_VALID(pRequest->self)); - tfree(pRequest->msgBuf); - tfree(pRequest->sqlstr); - tfree(pRequest->pInfo); - tfree(pRequest->pDb); + taosMemoryFreeClear(pRequest->msgBuf); + taosMemoryFreeClear(pRequest->sqlstr); + taosMemoryFreeClear(pRequest->pInfo); + taosMemoryFreeClear(pRequest->pDb); doFreeReqResultInfo(&pRequest->body.resInfo); qDestroyQueryPlan(pRequest->body.pDag); + if (pRequest->body.queryJob != 0) { + schedulerFreeJob(pRequest->body.queryJob); + } + if (pRequest->body.showInfo.pArray != NULL) { taosArrayDestroy(pRequest->body.showInfo.pArray); } deregisterRequest(pRequest); - tfree(pRequest); + taosMemoryFreeClear(pRequest); } void destroyRequest(SRequestObj *pRequest) { @@ -356,7 +360,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); } - free(charset); + taosMemoryFree(charset); } else { // it may be windows system tscInfo("charset remains:%s", tsCharset); } @@ -404,10 +408,10 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, str, TD_TIMEZONE_LEN); + tstrncpy(tsTimezoneStr, str, TD_TIMEZONE_LEN); tsSetTimeZone(); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); + tscDebug("timezone set:%s, input:%s by taos_options", tsTimezoneStr, str); } else { tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 479281eec6..abb6e7fbd1 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -91,7 +91,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } else { tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); + tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); tFreeSTableMetaBatchRsp(&batchMetaRsp); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -166,7 +166,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { static int32_t emptyRspNum = 0; if (code != 0) { - tfree(param); + taosMemoryFreeClear(param); return -1; } @@ -179,12 +179,12 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL || NULL == *pInst) { tscError("cluster not exist, key:%s", key); - tfree(param); + taosMemoryFreeClear(param); tFreeClientHbBatchRsp(&pRsp); return -1; } - tfree(param); + taosMemoryFreeClear(param); if (rspNum) { tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, @@ -317,7 +317,7 @@ void hbFreeReq(void *req) { } SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { - SClientHbBatchReq *pBatchReq = calloc(1, sizeof(SClientHbBatchReq)); + SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -346,7 +346,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { if (code) { taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq); - tfree(pBatchReq); + taosMemoryFreeClear(pBatchReq); } return pBatchReq; @@ -387,7 +387,7 @@ static void *hbThreadFunc(void *param) { continue; } int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); - void *buf = malloc(tlen); + void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq, false); @@ -396,13 +396,13 @@ static void *hbThreadFunc(void *param) { } tSerializeSClientHbBatchReq(buf, tlen, pReq); - SMsgSendInfo *pInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo *pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq, false); hbClearReqInfo(pAppHbMgr); - free(buf); + taosMemoryFree(buf); break; } pInfo->fp = hbAsyncCallBack; @@ -458,7 +458,7 @@ static void hbStopThread() { SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { hbMgrInit(); - SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr)); + SAppHbMgr *pAppHbMgr = taosMemoryMalloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -478,7 +478,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { if (pAppHbMgr->activeInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(pAppHbMgr); + taosMemoryFree(pAppHbMgr); return NULL; } @@ -488,7 +488,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { if (pAppHbMgr->connInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(pAppHbMgr); + taosMemoryFree(pAppHbMgr); return NULL; } @@ -580,7 +580,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3 switch (hbType) { case HEARTBEAT_TYPE_QUERY: { - int64_t *pClusterId = malloc(sizeof(int64_t)); + int64_t *pClusterId = taosMemoryMalloc(sizeof(int64_t)); *pClusterId = clusterId; info.param = pClusterId; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 47e92ed0ca..6e65a4267f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -100,7 +100,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); SAppInstInfo* p = NULL; if (pInst == NULL) { - p = calloc(1, sizeof(struct SAppInstInfo)); + p = taosMemoryCalloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); p->pAppHbMgr = appHbMgrInit(p, key); @@ -111,7 +111,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, taosThreadMutexUnlock(&appInfo.mutex); - tfree(key); + taosMemoryFreeClear(key); return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst); } @@ -122,7 +122,7 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj* return TSDB_CODE_TSC_OUT_OF_MEMORY; } - (*pRequest)->sqlstr = malloc(sqlLen + 1); + (*pRequest)->sqlstr = taosMemoryMalloc(sqlLen + 1); if ((*pRequest)->sqlstr == NULL) { tscError("0x%" PRIx64 " failed to prepare sql string buffer", (*pRequest)->self); (*pRequest)->msgBuf = strdup("failed to prepare sql string buffer"); @@ -199,7 +199,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .queryId = pRequest->requestId, .acctId = pRequest->pTscObj->acctId, .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), - .pAstRoot = pQuery->pRoot + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite }; int32_t code = qCreateQueryPlan(&cxt, pPlan, pNodeList); if (code != 0) { @@ -212,7 +213,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t assert(pSchema != NULL && numOfCols > 0); pResInfo->numOfCols = numOfCols; - pResInfo->fields = calloc(numOfCols, sizeof(pSchema[0])); + pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(pSchema[0])); for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { pResInfo->fields[i].bytes = pSchema[i].bytes; @@ -330,6 +331,8 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { pRequest->code = code; break; } + + destroyRequest(pRequest); } return pRequest; @@ -421,7 +424,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t } static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -441,14 +444,14 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { if (db != NULL) { tstrncpy(connectReq.db, db, sizeof(connectReq.db)); } - tfree(db); + taosMemoryFreeClear(db); connectReq.pid = htonl(appInfo.pid); connectReq.startTime = htobe64(appInfo.startTime); tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); - void* pReq = malloc(contLen); + void* pReq = taosMemoryMalloc(contLen); tSerializeSConnectReq(pReq, contLen, &connectReq); pMsgSendInfo->msgInfo.len = contLen; @@ -458,8 +461,8 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->msgInfo.pData); - tfree(pMsgBody); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + taosMemoryFreeClear(pMsgBody); } bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP || msgType == TDMT_VND_QUERY_HEARTBEAT_RSP; @@ -500,7 +503,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SDataBuf buf = {.len = pMsg->contLen, .pData = NULL, .handle = pMsg->handle}; if (pMsg->contLen > 0) { - buf.pData = calloc(1, pMsg->contLen); + buf.pData = taosMemoryCalloc(1, pMsg->contLen); if (buf.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pMsg->code = TSDB_CODE_OUT_OF_MEMORY; @@ -592,7 +595,7 @@ void* doFetchRow(SRequestObj* pRequest) { } SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); - SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + SVShowTablesReq* pShowReq = taosMemoryCalloc(1, sizeof(SVShowTablesReq)); pShowReq->head.vgId = htonl(pVgroupInfo->vgId); pRequest->body.requestMsg.len = sizeof(SVShowTablesReq); @@ -670,10 +673,10 @@ _return: static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { if (pResInfo->row == NULL) { - pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->pCol = calloc(pResInfo->numOfCols, sizeof(SResultColumn)); - pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); - pResInfo->convertBuf = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->row = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = taosMemoryCalloc(pResInfo->numOfCols, sizeof(SResultColumn)); + pResInfo->length = taosMemoryCalloc(pResInfo->numOfCols, sizeof(int32_t)); + pResInfo->convertBuf = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); if (pResInfo->row == NULL || pResInfo->pCol == NULL || pResInfo->length == NULL || pResInfo->convertBuf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -681,7 +684,7 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { for(int32_t i = 0; i < pResInfo->numOfCols; ++i) { if(pResInfo->fields[i].type == TSDB_DATA_TYPE_NCHAR) { - pResInfo->convertBuf[i] = calloc(1, NCHAR_WIDTH_TO_BYTES(pResInfo->fields[i].bytes)); + pResInfo->convertBuf[i] = taosMemoryCalloc(1, NCHAR_WIDTH_TO_BYTES(pResInfo->fields[i].bytes)); } } } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 9c2b23d740..4314391743 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -32,7 +32,7 @@ int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; setErrno(pRequest, code); - free(pMsg->pData); + taosMemoryFree(pMsg->pData); tsem_post(&pRequest->body.rspSem); return code; } @@ -40,7 +40,7 @@ int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { - free(pMsg->pData); + taosMemoryFree(pMsg->pData); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; @@ -77,13 +77,13 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, pTscObj->pAppInfo->numOfConns); - free(pMsg->pData); + taosMemoryFree(pMsg->pData); tsem_post(&pRequest->body.rspSem); return 0; } SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; @@ -96,13 +96,13 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { retrieveReq.showId = pRequest->body.showInfo.execId; int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq); - void* pReq = malloc(contLen); + void* pReq = taosMemoryMalloc(contLen); tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq); pMsgSendInfo->msgInfo.pData = pReq; pMsgSendInfo->msgInfo.len = contLen; pMsgSendInfo->msgInfo.handle = NULL; } else { - SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); + SVShowTablesFetchReq* pFetchMsg = taosMemoryCalloc(1, sizeof(SVShowTablesFetchReq)); if (pFetchMsg == NULL) { return NULL; } @@ -135,12 +135,12 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { tDeserializeSShowRsp(pMsg->pData, pMsg->len, &showRsp); STableMetaRsp *pMetaMsg = &showRsp.tableMeta; - tfree(pRequest->body.resInfo.pRspMsg); + taosMemoryFreeClear(pRequest->body.resInfo.pRspMsg); pRequest->body.resInfo.pRspMsg = pMsg->pData; SReqResultInfo* pResInfo = &pRequest->body.resInfo; if (pResInfo->fields == NULL) { - TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); + TAOS_FIELD* pFields = taosMemoryCalloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { SSchema* pSchema = &pMetaMsg->pSchemas[i]; tstrncpy(pFields[i].name, pSchema->name, tListLen(pFields[i].name)); @@ -171,7 +171,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj *pRequest = param; SReqResultInfo *pResInfo = &pRequest->body.resInfo; - tfree(pResInfo->pRspMsg); + taosMemoryFreeClear(pResInfo->pRspMsg); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -204,7 +204,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; SReqResultInfo* pResInfo = &pRequest->body.resInfo; - tfree(pResInfo->pRspMsg); + taosMemoryFreeClear(pResInfo->pRspMsg); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -237,7 +237,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { // todo rsp with the vnode id list SRequestObj* pRequest = param; - free(pMsg->pData); + taosMemoryFree(pMsg->pData); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); } @@ -266,7 +266,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { } if (code != TSDB_CODE_SUCCESS) { - free(pMsg->pData); + taosMemoryFree(pMsg->pData); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; @@ -284,7 +284,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (code != 0) { terrno = code; if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash); - tfree(output.dbVgroup); + taosMemoryFreeClear(output.dbVgroup); tscError("failed to build use db output since %s", terrstr()); } else { @@ -304,7 +304,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { tNameGetDbName(&name, db); setConnectionDB(pRequest->pTscObj, db); - free(pMsg->pData); + taosMemoryFree(pMsg->pData); tsem_post(&pRequest->body.rspSem); return 0; } @@ -313,7 +313,7 @@ int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { assert(pMsg != NULL && param != NULL); SRequestObj* pRequest = param; - free(pMsg->pData); + taosMemoryFree(pMsg->pData); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 49fe7aa653..2a9b3cdf64 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -27,9 +27,7 @@ #include "tref.h" struct tmq_list_t { - int32_t cnt; - int32_t tot; - char* elems[]; + SArray container; }; struct tmq_topic_vgroup_t { @@ -45,11 +43,14 @@ struct tmq_topic_vgroup_list_t { struct tmq_conf_t { char clientId[256]; char groupId[TSDB_CGROUP_LEN]; - int8_t auto_commit; + int8_t autoCommit; int8_t resetOffset; + uint16_t port; + char* ip; + char* user; + char* pass; + char* db; tmq_commit_cb* commit_cb; - /*char* ip;*/ - /*uint16_t port;*/ }; struct tmq_t { @@ -98,12 +99,13 @@ typedef struct { typedef struct { // subscribe info - int32_t sqlLen; - char* sql; - char* topicName; - int64_t topicId; - int32_t nextVgIdx; - SArray* vgs; // SArray + int32_t sqlLen; + char* sql; + char* topicName; + int64_t topicId; + int32_t nextVgIdx; + SArray* vgs; // SArray + SSchemaWrapper schema; } SMqClientTopic; typedef struct { @@ -136,14 +138,14 @@ typedef struct { } SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { - tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); - conf->auto_commit = false; + tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); + conf->autoCommit = false; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; return conf; } void tmq_conf_destroy(tmq_conf_t* conf) { - if (conf) free(conf); + if (conf) taosMemoryFree(conf); } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { @@ -151,21 +153,24 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value strcpy(conf->groupId, value); return TMQ_CONF_OK; } + if (strcmp(key, "client.id") == 0) { strcpy(conf->clientId, value); return TMQ_CONF_OK; } + if (strcmp(key, "enable.auto.commit") == 0) { if (strcmp(value, "true") == 0) { - conf->auto_commit = true; + conf->autoCommit = true; return TMQ_CONF_OK; } else if (strcmp(value, "false") == 0) { - conf->auto_commit = false; + conf->autoCommit = false; return TMQ_CONF_OK; } else { return TMQ_CONF_INVALID; } } + if (strcmp(key, "auto.offset.reset") == 0) { if (strcmp(value, "none") == 0) { conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; @@ -180,26 +185,49 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_INVALID; } } + + if (strcmp(key, "connection.ip") == 0) { + conf->ip = strdup(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "connection.user") == 0) { + conf->user = strdup(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "connection.pass") == 0) { + conf->pass = strdup(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "connection.port") == 0) { + conf->port = atoi(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "connection.db") == 0) { + conf->db = strdup(value); + return TMQ_CONF_OK; + } + return TMQ_CONF_UNKNOWN; } tmq_list_t* tmq_list_new() { - tmq_list_t* ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); - if (ptr == NULL) { - return ptr; - } - ptr->cnt = 0; - ptr->tot = 8; - return ptr; + // + return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); } -int32_t tmq_list_append(tmq_list_t* ptr, const char* src) { - if (ptr->cnt >= ptr->tot - 1) return -1; - ptr->elems[ptr->cnt] = strdup(src); - ptr->cnt++; +int32_t tmq_list_append(tmq_list_t* list, const char* src) { + SArray* container = &list->container; + char* topic = strdup(src); + if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } +void tmq_list_destroy(tmq_list_t* list) { + SArray* container = (SArray*)list; + /*taosArrayDestroy(container);*/ + taosArrayDestroyEx(container, (void (*)(void*))taosMemoryFree); +} + void tmqClearUnhandleMsg(tmq_t* tmq) { tmq_message_t* msg; while (1) { @@ -254,7 +282,7 @@ tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) { } tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - tmq_t* pTmq = calloc(sizeof(tmq_t), 1); + tmq_t* pTmq = taosMemoryCalloc(sizeof(tmq_t), 1); if (pTmq == NULL) { return NULL; } @@ -268,17 +296,57 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); - pTmq->autoCommit = conf->auto_commit; + pTmq->autoCommit = conf->autoCommit; pTmq->commit_cb = conf->commit_cb; pTmq->resetOffsetCfg = conf->resetOffset; - tsem_init(&pTmq->rspSem, 0, 0); - pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + if (pTmq->clientTopics == NULL) { + taosMemoryFree(pTmq); + return NULL; + } pTmq->mqueue = taosOpenQueue(); pTmq->qall = taosAllocateQall(); + + tsem_init(&pTmq->rspSem, 0, 0); + + return pTmq; +} + +tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); + if (pTmq == NULL) { + return NULL; + } + pTmq->pTscObj = taos_connect(conf->ip, conf->user, conf->pass, conf->db, conf->port); + + pTmq->inWaiting = 0; + pTmq->status = 0; + pTmq->pollCnt = 0; + pTmq->epoch = 0; + pTmq->waitingRequest = 0; + pTmq->readyRequest = 0; + // set conf + strcpy(pTmq->clientId, conf->clientId); + strcpy(pTmq->groupId, conf->groupId); + pTmq->autoCommit = conf->autoCommit; + pTmq->commit_cb = conf->commit_cb; + pTmq->resetOffsetCfg = conf->resetOffset; + + pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); + pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + if (pTmq->clientTopics == NULL) { + taosMemoryFree(pTmq); + return NULL; + } + + pTmq->mqueue = taosOpenQueue(); + pTmq->qall = taosAllocateQall(); + + tsem_init(&pTmq->rspSem, 0, 0); + return pTmq; } @@ -317,7 +385,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); tEncodeSMqCMCommitOffsetReq(&encoder, &req); int32_t tlen = encoder.pos; - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { tCoderClear(&encoder); return -1; @@ -333,7 +401,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tscError("failed to malloc request"); } - SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); if (pParam == NULL) { return -1; } @@ -361,7 +429,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in } tsem_destroy(&pParam->rspSem); - free(pParam); + taosMemoryFree(pParam); if (pArray) { taosArrayDestroy(pArray); @@ -372,7 +440,8 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { SRequestObj* pRequest = NULL; - int32_t sz = topic_list->cnt; + SArray* container = &topic_list->container; + int32_t sz = taosArrayGetSize(container); // destroy ex taosArrayDestroy(tmq->clientTopics); tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); @@ -384,7 +453,8 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { req.topicNames = taosArrayInit(sz, sizeof(void*)); for (int i = 0; i < sz; i++) { - char* topicName = topic_list->elems[i]; + /*char* topicName = topic_list->elems[i];*/ + char* topicName = taosArrayGetP(container, i); SName name = {0}; char* dbName = getDbOfConnection(tmq->pTscObj); @@ -394,7 +464,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); tNameFromString(&name, topicName, T_NAME_TABLE); - char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); + char* topicFname = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); if (topicFname == NULL) { goto _return; } @@ -405,11 +475,11 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); taosArrayPush(tmq->clientTopics, &topic); taosArrayPush(req.topicNames, &topicFname); - free(dbName); + taosMemoryFree(dbName); } int tlen = tSerializeSCMSubscribeReq(NULL, &req); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { goto _return; } @@ -508,7 +578,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa strcpy(req.outputSTbName, tbName); int tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { goto _return; } @@ -532,7 +602,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa tsem_wait(&pRequest->body.rspSem); _return: - tfree(astStr); + taosMemoryFreeClear(astStr); qDestroyQuery(pQueryNode); /*if (sendInfo != NULL) {*/ /*destroySendMsgInfo(sendInfo);*/ @@ -594,7 +664,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i tNameExtractFullName(&name, req.name); int tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { goto _return; } @@ -618,7 +688,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i tsem_wait(&pRequest->body.rspSem); _return: - tfree(astStr); + taosMemoryFreeClear(astStr); qDestroyQuery(pQueryNode); /*if (sendInfo != NULL) {*/ /*destroySendMsgInfo(sendInfo);*/ @@ -757,7 +827,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { #if 0 if (pParam->sync == 1) { - /**pParam->msg = malloc(sizeof(tmq_message_t));*/ + /**pParam->msg = taosMemoryMalloc(sizeof(tmq_message_t));*/ *pParam->msg = taosAllocateQitem(sizeof(tmq_message_t)); if (*pParam->msg) { memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead)); @@ -774,7 +844,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/ + /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/ tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); if (pRsp == NULL) { goto WRITE_QUEUE_FAIL; @@ -884,7 +954,7 @@ END: int32_t tmqAskEp(tmq_t* tmq, bool sync) { int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* req = malloc(tlen); + SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen); if (req == NULL) { tscError("failed to malloc get subscribe ep buf"); return -1; @@ -893,21 +963,21 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { req->epoch = htonl(tmq->epoch); strcpy(req->cgroup, tmq->groupId); - SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam)); + SMqAskEpCbParam* pParam = taosMemoryMalloc(sizeof(SMqAskEpCbParam)); if (pParam == NULL) { tscError("failed to malloc subscribe param"); - free(req); + taosMemoryFree(req); return -1; } pParam->tmq = tmq; pParam->sync = sync; tsem_init(&pParam->rspSem, 0, 0); - SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { tsem_destroy(&pParam->rspSem); - free(pParam); - free(req); + taosMemoryFree(pParam); + taosMemoryFree(req); return -1; } @@ -967,7 +1037,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo reqOffset = tmq->resetOffsetCfg; } - SMqPollReq* pReq = malloc(sizeof(SMqPollReq)); + SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq)); if (pReq == NULL) { return NULL; } @@ -1003,7 +1073,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { return NULL; } - SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // TODO: out of mem @@ -1016,7 +1086,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { pParam->msg = &msg; tsem_init(&pParam->rspSem, 0, 0); - SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { return NULL; } @@ -1071,9 +1141,9 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { tsem_post(&tmq->rspSem); return -1; } - SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { - free(pReq); + taosMemoryFree(pReq); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; @@ -1083,10 +1153,10 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { pParam->epoch = tmq->epoch; pParam->sync = 0; - SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); if (sendInfo == NULL) { - free(pReq); - free(pParam); + taosMemoryFree(pReq); + taosMemoryFree(pParam); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; @@ -1258,7 +1328,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { return NULL; } - SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); + SMqPollCbParam* param = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (param == NULL) { ASSERT(false); taosMsleep(blocking_time); @@ -1289,7 +1359,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tsem_wait(¶m->rspSem); tsem_destroy(¶m->rspSem); - free(param); + taosMemoryFree(param); if (tmq_message == NULL) { if (beginVgIdx == pTopic->nextVgIdx) { @@ -1331,7 +1401,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; - SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam)); if (pParam == NULL) { continue; } @@ -1360,7 +1430,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; SMqPollRsp* pRsp = &tmq_message->msg; tDeleteSMqConsumeRsp(pRsp); - /*free(tmq_message);*/ + /*taosMemoryFree(tmq_message);*/ taosFreeQitem(tmq_message); } @@ -1403,7 +1473,7 @@ char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; #if 0 tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { - tmq_t* pTmq = malloc(sizeof(tmq_t)); + tmq_t* pTmq = taosMemoryMalloc(sizeof(tmq_t)); if (pTmq == NULL) { return NULL; } @@ -1417,7 +1487,7 @@ tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->msgInfo.pData); - tfree(pMsgBody); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + taosMemoryFreeClear(pMsgBody); } #endif diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 67e7333597..c624e383fb 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -115,7 +115,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con newSize = newSize * 1.5; } - char* buf = realloc(pColumnInfoData->pData, newSize); + char* buf = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (buf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -178,7 +178,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c uint32_t total = numOfRow1 + numOfRow2; if (BitmapLen(numOfRow1) < BitmapLen(total)) { - char* tmp = realloc(pColumnInfoData->nullbitmap, BitmapLen(total)); + char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(total)); uint32_t extend = BitmapLen(total) - BitmapLen(numOfRow1); memset(tmp + BitmapLen(numOfRow1), 0, extend); pColumnInfoData->nullbitmap = tmp; @@ -218,7 +218,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - char* p = realloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); + char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); if (p == NULL) { // TODO } @@ -232,7 +232,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co uint32_t len = pSource->varmeta.length; uint32_t oldLen = pColumnInfoData->varmeta.length; if (pColumnInfoData->varmeta.allocLen < len + oldLen) { - char* tmp = realloc(pColumnInfoData->pData, len + oldLen); + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; } @@ -247,7 +247,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); int32_t newSize = (numOfRow1 + numOfRow2) * pColumnInfoData->info.bytes; - char* tmp = realloc(pColumnInfoData->pData, newSize); + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; } @@ -268,16 +268,16 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - char* p = realloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); + char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - pColumnInfoData->varmeta.offset = (int32_t*) p; + pColumnInfoData->varmeta.offset = (int32_t*)p; memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { - char* tmp = realloc(pColumnInfoData->pData, pSource->varmeta.length); + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -289,7 +289,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); pColumnInfoData->varmeta.length = pSource->varmeta.length; } else { - char* tmp = realloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); + char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -298,7 +298,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); int32_t newSize = numOfRows * pColumnInfoData->info.bytes; - tmp = realloc(pColumnInfoData->pData, newSize); + tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -349,7 +349,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); int32_t newSize = oldLen + newLen; - char* tmp = realloc(pCol2->pData, newSize); + char* tmp = taosMemoryRealloc(pCol2->pData, newSize); if (tmp != NULL) { pCol2->pData = tmp; colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows); @@ -453,7 +453,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 return NULL; } - SSDataBlock* pDst = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pDst == NULL) { return NULL; } @@ -470,10 +470,10 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) { SVarColAttr* pAttr = &colInfo.varmeta; - pAttr->offset = calloc(rowCount, sizeof(int32_t)); + pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t)); } else { - colInfo.nullbitmap = calloc(1, BitmapLen(rowCount)); - colInfo.pData = calloc(rowCount, colInfo.info.bytes); + colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount)); + colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes); } taosArrayPush(pDst->pDataBlock, &colInfo); @@ -562,7 +562,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (pCol->varmeta.allocLen < colLength) { - char* tmp = realloc(pCol->pData, colLength); + char* tmp = taosMemoryRealloc(pCol->pData, colLength); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -587,11 +587,11 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { if (pBlock->info.rowSize == 0) { size_t rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - rowSize += pColInfo->info.bytes; - } + size_t numOfCols = pBlock->info.numOfCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + rowSize += pColInfo->info.bytes; + } pBlock->info.rowSize = rowSize; } @@ -610,7 +610,7 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { } SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) { - SSchema* pSchema = calloc(pBlock->info.numOfCols, sizeof(SSchema)); + SSchema* pSchema = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(SSchema)); for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pSchema[i].bytes = pColInfoData->info.bytes; @@ -637,7 +637,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { rowSize += sizeof(int32_t); } else { - rowSize += 1/8.0; // one bit for each record + rowSize += 1 / 8.0; // one bit for each record } } @@ -819,7 +819,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { int32_t rows = pDataBlock->info.rows; int32_t numOfCols = pDataBlock->info.numOfCols; - SColumnInfoData* pCols = calloc(numOfCols, sizeof(SColumnInfoData)); + SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); if (pCols == NULL) { return NULL; } @@ -829,14 +829,14 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { pCols[i].info = pColInfoData->info; if (IS_VAR_DATA_TYPE(pCols[i].info.type)) { - pCols[i].varmeta.offset = calloc(rows, sizeof(int32_t)); - pCols[i].pData = calloc(1, pColInfoData->varmeta.length); + pCols[i].varmeta.offset = taosMemoryCalloc(rows, sizeof(int32_t)); + pCols[i].pData = taosMemoryCalloc(1, pColInfoData->varmeta.length); pCols[i].varmeta.length = pColInfoData->varmeta.length; pCols[i].varmeta.allocLen = pCols[i].varmeta.length; } else { - pCols[i].nullbitmap = calloc(1, BitmapLen(rows)); - pCols[i].pData = calloc(rows, pCols[i].info.bytes); + pCols[i].nullbitmap = taosMemoryCalloc(1, BitmapLen(rows)); + pCols[i].pData = taosMemoryCalloc(rows, pCols[i].info.bytes); } } @@ -851,22 +851,22 @@ static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { pColInfoData->info = pCols[i].info; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - tfree(pColInfoData->varmeta.offset); + taosMemoryFreeClear(pColInfoData->varmeta.offset); pColInfoData->varmeta = pCols[i].varmeta; } else { - tfree(pColInfoData->nullbitmap); + taosMemoryFreeClear(pColInfoData->nullbitmap); pColInfoData->nullbitmap = pCols[i].nullbitmap; } - tfree(pColInfoData->pData); + taosMemoryFreeClear(pColInfoData->pData); pColInfoData->pData = pCols[i].pData; } - tfree(pCols); + taosMemoryFreeClear(pCols); } static int32_t* createTupleIndex(size_t rows) { - int32_t* index = calloc(rows, sizeof(int32_t)); + int32_t* index = taosMemoryCalloc(rows, sizeof(int32_t)); if (index == NULL) { return NULL; } @@ -878,7 +878,7 @@ static int32_t* createTupleIndex(size_t rows) { return index; } -static void destroyTupleIndex(int32_t* index) { tfree(index); } +static void destroyTupleIndex(int32_t* index) { taosMemoryFreeClear(index); } static __compar_fn_t getComparFn(int32_t type, int32_t order) { switch (type) { @@ -1019,8 +1019,8 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* size_t len = sortValLengthPerRow * pBlock->info.rows; - char* buf = calloc(1, len); - SHelper* phelper = calloc(numOfRows, sizeof(SHelper)); + char* buf = taosMemoryCalloc(1, len); + SHelper* phelper = taosMemoryCalloc(numOfRows, sizeof(SHelper)); for (int32_t i = 0; i < numOfRows; ++i) { phelper[i].index = i; phelper[i].pData = buf + sortValLengthPerRow * i; @@ -1163,7 +1163,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo } if (IS_VAR_DATA_TYPE(pColumn->info.type)) { - char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); + char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1173,9 +1173,9 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo pColumn->varmeta.length = 0; pColumn->varmeta.allocLen = 0; - tfree(pColumn->pData); + taosMemoryFreeClear(pColumn->pData); } else { - char* tmp = realloc(pColumn->nullbitmap, BitmapLen(numOfRows)); + char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1183,7 +1183,7 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo pColumn->nullbitmap = tmp; memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); assert(pColumn->info.bytes); - tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); + tmp = taosMemoryRealloc(pColumn->pData, numOfRows * pColumn->info.bytes); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1214,14 +1214,14 @@ void* blockDataDestroy(SSDataBlock* pBlock) { } blockDestroyInner(pBlock); - tfree(pBlock); + taosMemoryFreeClear(pBlock); return NULL; } SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { int32_t numOfCols = pDataBlock->info.numOfCols; - SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.numOfCols = numOfCols; @@ -1318,12 +1318,101 @@ int32_t tEncodeDataBlocks(void** buf, const SArray* blocks) { return tlen; } -void* tDecodeDataBlocks(const void* buf, SArray* blocks) { +void* tDecodeDataBlocks(const void* buf, SArray** blocks) { int32_t sz; buf = taosDecodeFixedI32(buf, &sz); + + *blocks = taosArrayInit(sz, sizeof(SSDataBlock)); for (int32_t i = 0; i < sz; i++) { SSDataBlock pBlock = {0}; buf = tDecodeDataBlock(buf, &pBlock); + taosArrayPush(*blocks, &pBlock); } return (void*)buf; } + +static char* formatTimestamp(char* buf, int64_t val, int precision) { + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + /* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm* ptm = localtime(&tt); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} +void blockDebugShowData(const SArray* dataBlocks) { + char pBuf[128]; + int32_t sz = taosArrayGetSize(dataBlocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + for (int32_t j = 0; j < rows; j++) { + printf("|"); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + printf(" %25s |", pBuf); + break; + case TSDB_DATA_TYPE_INT: + printf(" %15d |", *(int32_t*)var); + break; + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t*)var); + break; + case TSDB_DATA_TYPE_BIGINT: + printf(" %15ld |", *(int64_t*)var); + break; + case TSDB_DATA_TYPE_UBIGINT: + printf(" %15lu |", *(uint64_t*)var); + break; + } + } + printf("\n"); + } + } +} + diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ed5441fe99..1b7157c49c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -37,7 +37,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { #endif if (pCol->spaceSize < spaceNeeded) { - void *ptr = realloc(pCol->pData, spaceNeeded); + void *ptr = taosMemoryRealloc(pCol->pData, spaceNeeded); if (ptr == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, strerror(errno)); return -1; @@ -66,7 +66,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { */ STSchema *tdDupSchema(const STSchema *pSchema) { int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); - STSchema *tSchema = (STSchema *)malloc(tlen); + STSchema *tSchema = (STSchema *)taosMemoryMalloc(tlen); if (tSchema == NULL) return NULL; memcpy((void *)tSchema, (void *)pSchema, tlen); @@ -106,12 +106,12 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { if (tdInitTSchemaBuilder(&schemaBuilder, version) < 0) return NULL; for (int i = 0; i < numOfCols; i++) { - int8_t type = 0; - int16_t colId = 0; - int16_t bytes = 0; + col_type_t type = 0; + col_id_t colId = 0; + col_bytes_t bytes = 0; buf = taosDecodeFixedI8(buf, &type); buf = taosDecodeFixedI16(buf, &colId); - buf = taosDecodeFixedI16(buf, &bytes); + buf = taosDecodeFixedI32(buf, &bytes); if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) { tdDestroyTSchemaBuilder(&schemaBuilder); return NULL; @@ -127,7 +127,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { if (pBuilder == NULL) return -1; pBuilder->tCols = 256; - pBuilder->columns = (STColumn *)malloc(sizeof(STColumn) * pBuilder->tCols); + pBuilder->columns = (STColumn *)taosMemoryMalloc(sizeof(STColumn) * pBuilder->tCols); if (pBuilder->columns == NULL) return -1; tdResetTSchemaBuilder(pBuilder, version); @@ -136,7 +136,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { if (pBuilder) { - tfree(pBuilder->columns); + taosMemoryFreeClear(pBuilder->columns); } } @@ -148,12 +148,12 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { pBuilder->version = version; } -int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes) { +int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes) { if (!isValidDataType(type)) return -1; if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; - STColumn *columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); + STColumn *columns = (STColumn *)taosMemoryRealloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); if (columns == NULL) return -1; pBuilder->columns = columns; } @@ -191,7 +191,7 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { int tlen = sizeof(STSchema) + sizeof(STColumn) * pBuilder->nCols; - STSchema *pSchema = (STSchema *)malloc(tlen); + STSchema *pSchema = (STSchema *)taosMemoryMalloc(tlen); if (pSchema == NULL) return NULL; schemaVersion(pSchema) = pBuilder->version; @@ -221,7 +221,7 @@ void tdInitDataRow(SDataRow row, STSchema *pSchema) { SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { int32_t size = dataRowMaxBytesFromSchema(pSchema); - SDataRow row = malloc(size); + SDataRow row = taosMemoryMalloc(size); if (row == NULL) return NULL; tdInitDataRow(row, pSchema); @@ -232,11 +232,11 @@ SDataRow tdNewDataRowFromSchema(STSchema *pSchema) { * Free the SDataRow object */ void tdFreeDataRow(SDataRow row) { - if (row) free(row); + if (row) taosMemoryFree(row); } SDataRow tdDataRowDup(SDataRow row) { - SDataRow trow = malloc(dataRowLen(row)); + SDataRow trow = taosMemoryMalloc(dataRowLen(row)); if (trow == NULL) return NULL; dataRowCpy(trow, row); @@ -244,7 +244,7 @@ SDataRow tdDataRowDup(SDataRow row) { } SMemRow tdMemRowDup(SMemRow row) { - SMemRow trow = malloc(memRowTLen(row)); + SMemRow trow = taosMemoryMalloc(memRowTLen(row)); if (trow == NULL) return NULL; memRowCpy(trow, row); @@ -348,7 +348,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) { } SDataCols *tdNewDataCols(int maxCols, int maxRows) { - SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols)); + SDataCols *pCols = (SDataCols *)taosMemoryCalloc(1, sizeof(SDataCols)); if (pCols == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno)); return NULL; @@ -360,7 +360,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->numOfCols = 0; if (maxCols > 0) { - pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); + pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol)); if (pCols->cols == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); @@ -384,7 +384,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { int oldMaxCols = pCols->maxCols; if (schemaNCols(pSchema) > oldMaxCols) { pCols->maxCols = schemaNCols(pSchema); - void *ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); + void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); if (ptr == NULL) return -1; pCols->cols = ptr; for (i = oldMaxCols; i < pCols->maxCols; i++) { @@ -411,12 +411,12 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { int maxCols = pCols->maxCols; for (i = 0; i < maxCols; i++) { SDataCol *pCol = &pCols->cols[i]; - tfree(pCol->pData); + taosMemoryFreeClear(pCol->pData); } - free(pCols->cols); + taosMemoryFree(pCols->cols); pCols->cols = NULL; } - free(pCols); + taosMemoryFree(pCols); } return NULL; } @@ -641,7 +641,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i #endif SKVRow tdKVRowDup(SKVRow row) { - SKVRow trow = malloc(kvRowLen(row)); + SKVRow trow = taosMemoryMalloc(kvRowLen(row)); if (trow == NULL) return NULL; kvRowCpy(trow, row); @@ -674,7 +674,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { int oRowCols = kvRowNCols(row); ASSERT(diff > 0); - nrow = malloc(nRowLen); + nrow = taosMemoryMalloc(nRowLen); if (nrow == NULL) return -1; kvRowSetLen(nrow, nRowLen); @@ -692,7 +692,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { tdSortKVRowByColIdx(nrow); *orow = nrow; - free(row); + taosMemoryFree(row); } else { ASSERT(((SColIdx *)ptr)->colId == colId); if (IS_VAR_DATA_TYPE(type)) { @@ -703,7 +703,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { } else { // need to reallocate the memory int16_t nlen = kvRowLen(row) + (varDataTLen(value) - varDataTLen(pOldVal)); ASSERT(nlen > 0); - nrow = malloc(nlen); + nrow = taosMemoryMalloc(nlen); if (nrow == NULL) return -1; kvRowSetLen(nrow, nlen); @@ -728,7 +728,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { } *orow = nrow; - free(row); + taosMemoryFree(row); } } else { memcpy(kvRowColVal(row, (SColIdx *)ptr), value, TYPE_BYTES[type]); @@ -757,21 +757,21 @@ void *tdDecodeKVRow(void *buf, SKVRow *row) { int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) { pBuilder->tCols = 128; pBuilder->nCols = 0; - pBuilder->pColIdx = (SColIdx *)malloc(sizeof(SColIdx) * pBuilder->tCols); + pBuilder->pColIdx = (SColIdx *)taosMemoryMalloc(sizeof(SColIdx) * pBuilder->tCols); if (pBuilder->pColIdx == NULL) return -1; pBuilder->alloc = 1024; pBuilder->size = 0; - pBuilder->buf = malloc(pBuilder->alloc); + pBuilder->buf = taosMemoryMalloc(pBuilder->alloc); if (pBuilder->buf == NULL) { - free(pBuilder->pColIdx); + taosMemoryFree(pBuilder->pColIdx); return -1; } return 0; } void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) { - tfree(pBuilder->pColIdx); - tfree(pBuilder->buf); + taosMemoryFreeClear(pBuilder->pColIdx); + taosMemoryFreeClear(pBuilder->buf); } void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { @@ -785,7 +785,7 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { tlen += TD_KV_ROW_HEAD_SIZE; - SKVRow row = malloc(tlen); + SKVRow row = taosMemoryMalloc(tlen); if (row == NULL) return NULL; kvRowSetNCols(row, pBuilder->nCols); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 179a50c422..efd790ade8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -122,7 +122,7 @@ bool tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name bool tsKeepOriginalColumnName = 0; -// long query death-lock +// kill long query bool tsDeadLockKillQuery = 0; // tsdb config @@ -303,7 +303,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { static int32_t taosAddSystemCfg(SConfig *pCfg) { SysNameInfo info = taosGetSysNameInfo(); - if (cfgAddTimezone(pCfg, "timezone", tsTimezone) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr) != 0) return -1; if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; @@ -431,12 +431,13 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { static void taosSetSystemCfg(SConfig *pCfg) { SConfigItem *pItem = cfgGetItem(pCfg, "timezone"); osSetTimezone(pItem->str); - uDebug("timezone format changed from %s to %s", pItem->str, tsTimezone); - cfgSetItem(pCfg, "timezone", tsTimezone, pItem->stype); + uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); + cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype); const char *locale = cfgGetItem(pCfg, "locale")->str; const char *charset = cfgGetItem(pCfg, "charset")->str; taosSetSystemLocale(locale, charset); + osSetSystemLocale(locale, charset); bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; taosSetConsoleEcho(enableCore); @@ -483,7 +484,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc) { - osInit(); + osDefaultInit(); SConfig *pCfg = cfgInit(); if (pCfg == NULL) return -1; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 32e474a58c..f464ce6f50 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -299,14 +299,14 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); } tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols); for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].colId); + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); } @@ -333,7 +333,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols); for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); } @@ -371,24 +371,24 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { case TD_SUPER_TABLE: buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid)); buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); - pReq->stbCfg.pSchema = (SSchema *)malloc(pReq->stbCfg.nCols * sizeof(SSchema)); + pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); - buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); + buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId)); buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); } buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols); - pReq->stbCfg.pTagSchema = (SSchema *)malloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); + pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); - buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); + buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); } buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); if (pReq->stbCfg.nBSmaCols > 0) { - pReq->stbCfg.pBSmaCols = (col_id_t *)malloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t)); + pReq->stbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t)); for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) { buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i); } @@ -396,7 +396,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { pReq->stbCfg.pBSmaCols = NULL; } if (pReq->rollup) { - pReq->stbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam)); + pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); SRSmaParam *param = pReq->stbCfg.pRSmaParam; buf = taosDecodeFixedU32(buf, (uint32_t *)¶m->xFilesFactor); buf = taosDecodeFixedI8(buf, ¶m->delayUnit); @@ -419,16 +419,16 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { break; case TD_NORMAL_TABLE: buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols); - pReq->ntbCfg.pSchema = (SSchema *)malloc(pReq->ntbCfg.nCols * sizeof(SSchema)); + pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); - buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); + buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); } buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); if (pReq->ntbCfg.nBSmaCols > 0) { - pReq->ntbCfg.pBSmaCols = (col_id_t *)malloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t)); + pReq->ntbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t)); for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) { buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i); } @@ -436,7 +436,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { pReq->ntbCfg.pBSmaCols = NULL; } if (pReq->rollup) { - pReq->ntbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam)); + pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); SRSmaParam *param = pReq->ntbCfg.pRSmaParam; buf = taosDecodeFixedU32(buf, (uint32_t *)¶m->xFilesFactor); buf = taosDecodeFixedI8(buf, ¶m->delayUnit); @@ -608,7 +608,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR } if (pReq->commentLen > 0) { - pReq->comment = malloc(pReq->commentLen); + pReq->comment = taosMemoryMalloc(pReq->commentLen); if (pReq->comment == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; } @@ -623,7 +623,7 @@ void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { taosArrayDestroy(pReq->pColumns); taosArrayDestroy(pReq->pTags); taosArrayDestroy(pReq->pSmas); - tfree(pReq->comment); + taosMemoryFreeClear(pReq->comment); pReq->pColumns = NULL; pReq->pTags = NULL; pReq->pSmas = NULL; @@ -770,22 +770,22 @@ int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pR if (tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &pReq->astLen) < 0) return -1; if (pReq->exprLen > 0) { - pReq->expr = malloc(pReq->exprLen); + pReq->expr = taosMemoryMalloc(pReq->exprLen); if (pReq->expr == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->expr) < 0) return -1; } if (pReq->tagsFilterLen > 0) { - pReq->tagsFilter = malloc(pReq->tagsFilterLen); + pReq->tagsFilter = taosMemoryMalloc(pReq->tagsFilterLen); if (pReq->tagsFilter == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->tagsFilter) < 0) return -1; } if (pReq->sqlLen > 0) { - pReq->sql = malloc(pReq->sqlLen); + pReq->sql = taosMemoryMalloc(pReq->sqlLen); if (pReq->sql == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } if (pReq->astLen > 0) { - pReq->ast = malloc(pReq->astLen); + pReq->ast = taosMemoryMalloc(pReq->astLen); if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } @@ -796,10 +796,10 @@ int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pR } void tFreeSMCreateSmaReq(SMCreateSmaReq *pReq) { - tfree(pReq->expr); - tfree(pReq->tagsFilter); - tfree(pReq->sql); - tfree(pReq->ast); + taosMemoryFreeClear(pReq->expr); + taosMemoryFreeClear(pReq->tagsFilter); + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->ast); } int32_t tSerializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) { @@ -1987,7 +1987,7 @@ int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; if (pReq->payloadLen > 0) { - pReq->payload = malloc(pReq->payloadLen); + pReq->payload = taosMemoryMalloc(pReq->payloadLen); if (pReq->payload == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; } @@ -1997,7 +1997,7 @@ int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { return 0; } -void tFreeSShowReq(SShowReq *pReq) { tfree(pReq->payload); } +void tFreeSShowReq(SShowReq *pReq) { taosMemoryFreeClear(pReq->payload); } int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq *pReq) { SCoder encoder = {0}; @@ -2071,7 +2071,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1; int32_t totalCols = pRsp->numOfTags + pRsp->numOfColumns; - pRsp->pSchemas = malloc(sizeof(SSchema) * totalCols); + pRsp->pSchemas = taosMemoryMalloc(sizeof(SSchema) * totalCols); if (pRsp->pSchemas == NULL) return -1; for (int32_t i = 0; i < totalCols; ++i) { @@ -2152,7 +2152,7 @@ int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatc return 0; } -void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { tfree(pRsp->pSchemas); } +void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } void tFreeSTableMetaBatchRsp(STableMetaBatchRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); @@ -2304,13 +2304,13 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (sqlLen > 0) { - pReq->sql = calloc(1, sqlLen + 1); + pReq->sql = taosMemoryCalloc(1, sqlLen + 1); if (pReq->sql == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } if (astLen > 0) { - pReq->ast = calloc(1, astLen + 1); + pReq->ast = taosMemoryCalloc(1, astLen + 1); if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } @@ -2322,8 +2322,8 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR } void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { - tfree(pReq->sql); - tfree(pReq->ast); + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->ast); } int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) { @@ -2871,7 +2871,6 @@ int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pR tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeU64(&encoder, pRsp->seqId) < 0) return -1; if (tEncodeI32(&encoder, pRsp->epId.nodeId) < 0) return -1; if (tEncodeU16(&encoder, pRsp->epId.ep.port) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->epId.ep.fqdn) < 0) return -1; @@ -2900,7 +2899,6 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp * tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeU64(&decoder, &pRsp->seqId) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->epId.nodeId) < 0) return -1; if (tDecodeU16(&decoder, &pRsp->epId.ep.port) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->epId.ep.fqdn) < 0) return -1; @@ -3074,13 +3072,13 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (sqlLen > 0) { - pReq->sql = calloc(1, sqlLen + 1); + pReq->sql = taosMemoryCalloc(1, sqlLen + 1); if (pReq->sql == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } if (astLen > 0) { - pReq->ast = calloc(1, astLen + 1); + pReq->ast = taosMemoryCalloc(1, astLen + 1); if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } @@ -3091,51 +3089,6 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea } void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { - tfree(pReq->sql); - tfree(pReq->ast); -} - -int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) { - /*if (tStartEncode(pEncoder) < 0) return -1;*/ - if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->level) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->parallelizable) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->nextOpDst) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1; - if (pTask->sinkType == STREAM_SINK_TYPE__ASSIGNED) { - if (tEncodeI32(pEncoder, pTask->sinkVgId) < 0) return -1; - if (tEncodeSEpSet(pEncoder, &pTask->NextOpEp) < 0) return -1; - } - if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1; - /*tEndEncode(pEncoder);*/ - return pEncoder->pos; -} - -int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) { - /*if (tStartDecode(pDecoder) < 0) return -1;*/ - if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->parallelizable) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->nextOpDst) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1; - if (pTask->sinkType == STREAM_SINK_TYPE__ASSIGNED) { - if (tDecodeI32(pDecoder, &pTask->sinkVgId) < 0) return -1; - if (tDecodeSEpSet(pDecoder, &pTask->NextOpEp) < 0) return -1; - } - if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1; - /*tEndDecode(pDecoder);*/ - return 0; -} - -void tFreeSStreamTask(SStreamTask *pTask) { - // TODO - /*free(pTask->qmsg);*/ - /*free(pTask->executor);*/ - /*free(pTask);*/ + taosMemoryFreeClear(pReq->sql); + taosMemoryFreeClear(pReq->ast); } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 5561eb93c3..f4755f5b5e 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -30,13 +30,13 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil return NULL; } - SColumnFilterInfo* pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); + SColumnFilterInfo* pFilter = taosMemoryCalloc(1, numOfFilters * sizeof(SColumnFilterInfo)); memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); for (int32_t j = 0; j < numOfFilters; ++j) { if (pFilter[j].filterstr) { size_t len = (size_t) pFilter[j].len + 1 * TSDB_NCHAR_SIZE; - pFilter[j].pz = (int64_t) calloc(1, len); + pFilter[j].pz = (int64_t) taosMemoryCalloc(1, len); memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t) pFilter[j].len); } @@ -171,7 +171,7 @@ bool tNameIsValid(const SName* name) { SName* tNameDup(const SName* name) { assert(name != NULL); - SName* p = malloc(sizeof(SName)); + SName* p = taosMemoryMalloc(sizeof(SName)); memcpy(p, name, sizeof(SName)); return p; } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 48a63799f5..9ee2ec1300 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -119,7 +119,7 @@ int trbWriteCol(SRowBuilder *pRB, void *pData, col_id_t cid) { #endif STSRow *tdRowDup(STSRow *row) { - STSRow *trow = malloc(TD_ROW_LEN(row)); + STSRow *trow = taosMemoryMalloc(TD_ROW_LEN(row)); if (trow == NULL) return NULL; tdRowCpy(trow, row); diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 5f0a353226..15e741d307 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -30,7 +30,7 @@ static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); * @return */ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); + STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf)); if (pTSBuf == NULL) { return NULL; } @@ -41,7 +41,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (pTSBuf->pFile == NULL) { - free(pTSBuf); + taosMemoryFree(pTSBuf); return NULL; } @@ -66,7 +66,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { - STSBuf* pTSBuf = calloc(1, sizeof(STSBuf)); + STSBuf* pTSBuf = taosMemoryCalloc(1, sizeof(STSBuf)); if (pTSBuf == NULL) { return NULL; } @@ -78,7 +78,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); if (pTSBuf->pFile == NULL) { - free(pTSBuf); + taosMemoryFree(pTSBuf); return NULL; } @@ -101,7 +101,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { if (header.numOfGroup > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfGroup; - STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); + STSGroupBlockInfoEx* tmp = taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -122,7 +122,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize); + STSGroupBlockInfo* buf = (STSGroupBlockInfo*)taosMemoryCalloc(1, infoSize); if (buf == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -137,7 +137,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); } - free(buf); + taosMemoryFree(buf); ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); UNUSED(ret); @@ -166,11 +166,11 @@ void* tsBufDestroy(STSBuf* pTSBuf) { return NULL; } - tfree(pTSBuf->assistBuf); - tfree(pTSBuf->tsData.rawBuf); + taosMemoryFreeClear(pTSBuf->assistBuf); + taosMemoryFreeClear(pTSBuf->tsData.rawBuf); - tfree(pTSBuf->pData); - tfree(pTSBuf->block.payload); + taosMemoryFreeClear(pTSBuf->pData); + taosMemoryFreeClear(pTSBuf->block.payload); if (!pTSBuf->remainOpen) { taosCloseFile(&pTSBuf->pFile); @@ -184,7 +184,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { } taosVariantDestroy(&pTSBuf->block.tag); - free(pTSBuf); + taosMemoryFree(pTSBuf); return NULL; } @@ -200,7 +200,7 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); assert((int32_t)newSize > pTSBuf->numOfAlloc); - STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)taosMemoryRealloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return NULL; } @@ -240,7 +240,7 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { static void shrinkBuffer(STSList* ptsData) { // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size if (ptsData->allocSize >= ptsData->threshold * 2) { - char* rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE); + char* rawBuf = taosMemoryRealloc(ptsData->rawBuf, MEM_BUF_SIZE); if (rawBuf) { ptsData->rawBuf = rawBuf; ptsData->allocSize = MEM_BUF_SIZE; @@ -322,7 +322,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { static void expandBuffer(STSList* ptsData, int32_t inputSize) { if (ptsData->allocSize - ptsData->len < inputSize) { int32_t newSize = inputSize + ptsData->len; - char* tmp = realloc(ptsData->rawBuf, (size_t)newSize); + char* tmp = taosMemoryRealloc(ptsData->rawBuf, (size_t)newSize); if (tmp == NULL) { // todo } @@ -366,7 +366,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // NOTE: mix types tags are not supported size_t sz = 0; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); + char* tp = taosMemoryRealloc(pBlock->tag.pz, pBlock->tag.nLen + 1); assert(tp != NULL); memset(tp, 0, pBlock->tag.nLen + 1); @@ -812,7 +812,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { if (pDestBuf->numOfAlloc < newSize) { pDestBuf->numOfAlloc = newSize; - STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); + STSGroupBlockInfoEx* tmp = taosMemoryRealloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return -1; } @@ -1028,13 +1028,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { const int32_t INITIAL_GROUPINFO_SIZE = 4; pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; - pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); + pTSBuf->pData = taosMemoryCalloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); if (pTSBuf->pData == NULL) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); + pTSBuf->tsData.rawBuf = taosMemoryMalloc(MEM_BUF_SIZE); if (pTSBuf->tsData.rawBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -1044,13 +1044,13 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { pTSBuf->tsData.threshold = MEM_BUF_SIZE; pTSBuf->tsData.allocSize = MEM_BUF_SIZE; - pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); + pTSBuf->assistBuf = taosMemoryMalloc(MEM_BUF_SIZE); if (pTSBuf->assistBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->block.payload = malloc(MEM_BUF_SIZE); + pTSBuf->block.payload = taosMemoryMalloc(MEM_BUF_SIZE); if (pTSBuf->block.payload == NULL) { tsBufDestroy(pTSBuf); return NULL; @@ -1079,7 +1079,7 @@ void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { return; } - (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); + (*id) = taosMemoryMalloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); for (int32_t i = 0; i < size; ++i) { (*id)[i] = pTSBuf->pData[i].info.id; diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index c6aa1cb81d..3995db89b6 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -199,14 +199,14 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; - pVar->ucs4 = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); + pVar->ucs4 = taosMemoryCalloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); memcpy(pVar->ucs4, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; } case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method - pVar->pz = calloc(len + 1, sizeof(char)); + pVar->pz = taosMemoryCalloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); pVar->nLen = (int32_t)len; break; @@ -224,7 +224,7 @@ void taosVariantDestroy(SVariant *pVar) { if (pVar == NULL) return; if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) { - tfree(pVar->pz); + taosMemoryFreeClear(pVar->pz); pVar->nLen = 0; } @@ -233,7 +233,7 @@ void taosVariantDestroy(SVariant *pVar) { size_t num = taosArrayGetSize(pVar->arr); for (size_t i = 0; i < num; i++) { void *p = taosArrayGetP(pVar->arr, i); - free(p); + taosMemoryFree(p); } taosArrayDestroy(pVar->arr); pVar->arr = NULL; @@ -254,7 +254,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { pDst->nType = pSrc->nType; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; - char *p = realloc(pDst->pz, len); + char *p = taosMemoryRealloc(pDst->pz, len); assert(p); memset(p, 0, len); @@ -402,18 +402,18 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { // it is a in-place convert type for SVariant, local buffer is needed if (*pDest == pVariant->pz) { - pBuf = calloc(1, INITIAL_ALLOC_SIZE); + pBuf = taosMemoryCalloc(1, INITIAL_ALLOC_SIZE); } if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { size_t newSize = pVariant->nLen * TSDB_NCHAR_SIZE; if (pBuf != NULL) { if (newSize >= INITIAL_ALLOC_SIZE) { - pBuf = realloc(pBuf, newSize + 1); + pBuf = taosMemoryRealloc(pBuf, newSize + 1); } taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, pBuf); - free(pVariant->ucs4); + taosMemoryFree(pVariant->ucs4); pBuf[newSize] = 0; } else { taosUcs4ToMbs(pVariant->ucs4, (int32_t)newSize, *pDest); @@ -460,23 +460,23 @@ static int32_t toNchar(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } if (*pDest == pVariant->pz) { - TdUcs4 *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); + TdUcs4 *pWStr = taosMemoryCalloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); bool ret = taosMbsToUcs4(pDst, nLen, pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); if (!ret) { - tfree(pWStr); + taosMemoryFreeClear(pWStr); return -1; } // free the binary buffer in the first place if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { - free(pVariant->ucs4); + taosMemoryFree(pVariant->ucs4); } pVariant->ucs4 = pWStr; *pDestSize = taosUcs4len(pVariant->ucs4); // shrink the allocate memory, no need to check here. - char *tmp = realloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE); + char *tmp = taosMemoryRealloc(pVariant->ucs4, (*pDestSize + 1) * TSDB_NCHAR_SIZE); assert(tmp != NULL); pVariant->ucs4 = (TdUcs4 *)tmp; @@ -526,7 +526,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result if (token.type == TK_NULL) { if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -547,7 +547,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result } if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -566,7 +566,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result if (token.type == TK_FLOAT) { double v = wcstod(pVariant->ucs4, &endPtr); if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -577,7 +577,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result *result = (int64_t)v; } else if (token.type == TK_NULL) { if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } setNull((char *)result, type, tDataTypes[type].bytes); @@ -585,7 +585,7 @@ static FORCE_INLINE int32_t convertToInteger(SVariant *pVariant, int64_t *result } else { int64_t val = wcstoll(pVariant->ucs4, &endPtr, 10); if (releaseVariantPtr) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->nLen = 0; } @@ -971,21 +971,21 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { errno = 0; double v = strtod(pVariant->pz, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); return -1; } - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->d = v; } else if (pVariant->nType == TSDB_DATA_TYPE_NCHAR) { errno = 0; double v = wcstod(pVariant->ucs4, NULL); if ((errno == ERANGE && v == -1) || (isinf(v) || isnan(v))) { - free(pVariant->pz); + taosMemoryFree(pVariant->pz); return -1; } - free(pVariant->pz); + taosMemoryFree(pVariant->pz); pVariant->d = v; } else if (pVariant->nType >= TSDB_DATA_TYPE_BOOL && pVariant->nType <= TSDB_DATA_TYPE_BIGINT) { double tmp = (double)pVariant->i; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index d5fb61929a..ccd800d3f4 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -99,7 +99,7 @@ TEST(testCase, toInteger_test) { } TEST(testCase, Datablock_test) { - SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); b->info.numOfCols = 2; b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -108,8 +108,8 @@ TEST(testCase, Datablock_test) { infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.colId = 1; - infoData.pData = (char*) calloc(40, infoData.info.bytes); - infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (40/8)); + infoData.pData = (char*) taosMemoryCalloc(40, infoData.info.bytes); + infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (40/8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -117,7 +117,7 @@ TEST(testCase, Datablock_test) { infoData1.info.type = TSDB_DATA_TYPE_BINARY; infoData1.info.colId = 2; - infoData1.varmeta.offset = (int32_t*) calloc(40, sizeof(uint32_t)); + infoData1.varmeta.offset = (int32_t*) taosMemoryCalloc(40, sizeof(uint32_t)); taosArrayPush(b->pDataBlock, &infoData1); char* str = "the value of: %d"; @@ -178,7 +178,7 @@ TEST(testCase, Datablock_test) { #if 0 TEST(testCase, non_var_dataBlock_split_test) { - SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); b->info.numOfCols = 2; b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -189,8 +189,8 @@ TEST(testCase, non_var_dataBlock_split_test) { int32_t numOfRows = 1000000; - infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + infoData.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -198,8 +198,8 @@ TEST(testCase, non_var_dataBlock_split_test) { infoData1.info.type = TSDB_DATA_TYPE_TINYINT; infoData1.info.colId = 2; - infoData1.pData = (char*) calloc(numOfRows, infoData.info.bytes); - infoData1.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + infoData1.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData1.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); taosArrayPush(b->pDataBlock, &infoData1); for(int32_t i = 0; i < numOfRows; ++i) { @@ -233,7 +233,7 @@ TEST(testCase, non_var_dataBlock_split_test) { #endif TEST(testCase, var_dataBlock_split_test) { - SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); b->info.numOfCols = 2; b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -244,8 +244,8 @@ TEST(testCase, var_dataBlock_split_test) { infoData.info.type = TSDB_DATA_TYPE_INT; infoData.info.colId = 1; - infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + infoData.pData = (char*) taosMemoryCalloc(numOfRows, infoData.info.bytes); + infoData.nullbitmap = (char*) taosMemoryCalloc(1, sizeof(char) * (numOfRows/8)); taosArrayPush(b->pDataBlock, &infoData); SColumnInfoData infoData1 = {0}; @@ -253,7 +253,7 @@ TEST(testCase, var_dataBlock_split_test) { infoData1.info.type = TSDB_DATA_TYPE_BINARY; infoData1.info.colId = 2; - infoData1.varmeta.offset = (int32_t*) calloc(numOfRows, sizeof(uint32_t)); + infoData1.varmeta.offset = (int32_t*) taosMemoryCalloc(numOfRows, sizeof(uint32_t)); taosArrayPush(b->pDataBlock, &infoData1); char buf[41] = {0}; diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c index 4236e85a7f..b9c41ebf43 100644 --- a/source/dnode/bnode/src/bnode.c +++ b/source/dnode/bnode/src/bnode.c @@ -16,12 +16,12 @@ #include "bndInt.h" SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) { - SBnode *pBnode = calloc(1, sizeof(SBnode)); + SBnode *pBnode = taosMemoryCalloc(1, sizeof(SBnode)); pBnode->msgCb = pOption->msgCb; return pBnode; } -void bndClose(SBnode *pBnode) { free(pBnode); } +void bndClose(SBnode *pBnode) { taosMemoryFree(pBnode); } int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } diff --git a/source/dnode/mgmt/bnode/src/bmInt.c b/source/dnode/mgmt/bnode/src/bmInt.c index 34c6dbeb57..e2506ab383 100644 --- a/source/dnode/mgmt/bnode/src/bmInt.c +++ b/source/dnode/mgmt/bnode/src/bmInt.c @@ -73,7 +73,7 @@ int32_t bmDrop(SMgmtWrapper *pWrapper) { bmCloseImp(pMgmt); taosRemoveDir(pMgmt->path); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("bnode-mgmt is dropped"); return 0; } @@ -85,13 +85,13 @@ static void bmClose(SMgmtWrapper *pWrapper) { dInfo("bnode-mgmt start to cleanup"); bmCloseImp(pMgmt); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("bnode-mgmt is cleaned up"); } int32_t bmOpen(SMgmtWrapper *pWrapper) { dInfo("bnode-mgmt start to init"); - SBnodeMgmt *pMgmt = calloc(1, sizeof(SBnodeMgmt)); + SBnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SBnodeMgmt)); if (pMgmt == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index a6c9852546..24c7ad7968 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -139,8 +139,8 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) { dTrace("msg:%p, get from parent queue", pRsp); pRsp->pCont = pCont; - dndSendRsp(pWrapper, pRsp); - free(pRsp); + dndSendRpcRsp(pWrapper, pRsp); + taosMemoryFree(pRsp); } static int32_t dndRunInMultiProcess(SDnode *pDnode) { @@ -175,8 +175,8 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) { .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, .parentQueueSize = 1024 * 1024 * 2, // size will be a configuration item .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, - .parentdMallocHeadFp = (ProcMallocFp)malloc, - .parentFreeHeadFp = (ProcFreeFp)free, + .parentdMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, + .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, .pParent = pWrapper, diff --git a/source/dnode/mgmt/container/src/dndInt.c b/source/dnode/mgmt/container/src/dndInt.c index ab19415168..33d6bb0ee3 100644 --- a/source/dnode/mgmt/container/src/dndInt.c +++ b/source/dnode/mgmt/container/src/dndInt.c @@ -134,6 +134,7 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { dndGetStartup(pDnode, pStartup); dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)}; + SRpcMsg rpcRsp = { + .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; rpcSendResponse(&rpcRsp); } diff --git a/source/dnode/mgmt/container/src/dndObj.c b/source/dnode/mgmt/container/src/dndObj.c index 2123f5946b..b9ea8df808 100644 --- a/source/dnode/mgmt/container/src/dndObj.c +++ b/source/dnode/mgmt/container/src/dndObj.c @@ -39,19 +39,19 @@ static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) { static void dndClearMemory(SDnode *pDnode) { for (ENodeType n = 0; n < NODE_MAX; ++n) { SMgmtWrapper *pMgmt = &pDnode->wrappers[n]; - tfree(pMgmt->path); + taosMemoryFreeClear(pMgmt->path); } if (pDnode->pLockFile != NULL) { taosUnLockFile(pDnode->pLockFile); taosCloseFile(&pDnode->pLockFile); pDnode->pLockFile = NULL; } - tfree(pDnode->localEp); - tfree(pDnode->localFqdn); - tfree(pDnode->firstEp); - tfree(pDnode->secondEp); - tfree(pDnode->dataDir); - free(pDnode); + taosMemoryFreeClear(pDnode->localEp); + taosMemoryFreeClear(pDnode->localFqdn); + taosMemoryFreeClear(pDnode->firstEp); + taosMemoryFreeClear(pDnode->secondEp); + taosMemoryFreeClear(pDnode->dataDir); + taosMemoryFree(pDnode); dDebug("dnode object memory is cleared, data:%p", pDnode); } @@ -61,7 +61,7 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { char path[PATH_MAX] = {0}; SDnode *pDnode = NULL; - pDnode = calloc(1, sizeof(SDnode)); + pDnode = taosMemoryCalloc(1, sizeof(SDnode)); if (pDnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; diff --git a/source/dnode/mgmt/dnode/src/dmFile.c b/source/dnode/mgmt/dnode/src/dmFile.c index 9acfc2960f..d44b1222a3 100644 --- a/source/dnode/mgmt/dnode/src/dmFile.c +++ b/source/dnode/mgmt/dnode/src/dmFile.c @@ -24,7 +24,7 @@ int32_t dmReadFile(SDnodeMgmt *pMgmt) { int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; int32_t len = 0; int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); cJSON *root = NULL; char file[PATH_MAX]; TdFilePtr pFile = NULL; @@ -134,7 +134,7 @@ int32_t dmReadFile(SDnodeMgmt *pMgmt) { dmPrintDnodes(pMgmt); PRASE_DNODE_OVER: - if (content != NULL) free(content); + if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); @@ -171,7 +171,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) { int32_t len = 0; int32_t maxLen = 256 * 1024; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pDnode->dnodeId); @@ -197,7 +197,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) { taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); - free(content); + taosMemoryFree(content); char realfile[PATH_MAX]; snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); diff --git a/source/dnode/mgmt/dnode/src/dmInt.c b/source/dnode/mgmt/dnode/src/dmInt.c index 8aa87cb7bc..53049f7e78 100644 --- a/source/dnode/mgmt/dnode/src/dmInt.c +++ b/source/dnode/mgmt/dnode/src/dmInt.c @@ -80,7 +80,7 @@ static int32_t dmStart(SMgmtWrapper *pWrapper) { int32_t dmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; - SDnodeMgmt *pMgmt = calloc(1, sizeof(SDnodeMgmt)); + SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); dInfo("dnode-mgmt start to init"); pDnode->dnodeId = 0; @@ -138,7 +138,7 @@ void dmCleanup(SMgmtWrapper *pWrapper) { taosWUnLockLatch(&pMgmt->latch); - free(pMgmt); + taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; dInfo("dnode-mgmt is cleaned up"); } diff --git a/source/dnode/mgmt/dnode/src/dmMsg.c b/source/dnode/mgmt/dnode/src/dmMsg.c index 836817e772..eb4e843c55 100644 --- a/source/dnode/mgmt/dnode/src/dmMsg.c +++ b/source/dnode/mgmt/dnode/src/dmMsg.c @@ -36,7 +36,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); diff --git a/source/dnode/mgmt/dnode/src/dmWorker.c b/source/dnode/mgmt/dnode/src/dmWorker.c index be08aad7cb..d34a26436c 100644 --- a/source/dnode/mgmt/dnode/src/dmWorker.c +++ b/source/dnode/mgmt/dnode/src/dmWorker.c @@ -23,7 +23,7 @@ static void *dmThreadRoutine(void *param) { SDnodeMgmt *pMgmt = param; - SDnode *pDnode = pMgmt->pDnode; + SDnode * pDnode = pMgmt->pDnode; int64_t lastStatusTime = taosGetTimestampMs(); int64_t lastMonitorTime = lastStatusTime; @@ -55,7 +55,7 @@ static void *dmThreadRoutine(void *param) { static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pInfo->ahandle; - SDnode *pDnode = pMgmt->pDnode; + SDnode * pDnode = pMgmt->pDnode; SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; dTrace("msg:%p, will be processed in dnode queue", pMsg); diff --git a/source/dnode/mgmt/mnode/inc/mmInt.h b/source/dnode/mgmt/mnode/inc/mmInt.h index d57088474f..cd4585048b 100644 --- a/source/dnode/mgmt/mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mnode/inc/mmInt.h @@ -28,6 +28,7 @@ typedef struct SMnodeMgmt { SDnode *pDnode; SMgmtWrapper *pWrapper; const char *path; + SSingleWorker queryWorker; SSingleWorker readWorker; SSingleWorker writeWorker; SSingleWorker syncWorker; @@ -57,11 +58,13 @@ void mmStopWorker(SMnodeMgmt *pMgmt); int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmProcessQueryMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg); int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpcMsg); +int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); #ifdef __cplusplus } #endif -#endif /*_TD_DND_MNODE_INT_H_*/ \ No newline at end of file +#endif /*_TD_DND_MNODE_INT_H_*/ diff --git a/source/dnode/mgmt/mnode/src/mmFile.c b/source/dnode/mgmt/mnode/src/mmFile.c index 757b397473..e5cc0ce087 100644 --- a/source/dnode/mgmt/mnode/src/mmFile.c +++ b/source/dnode/mgmt/mnode/src/mmFile.c @@ -20,7 +20,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; int32_t len = 0; int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); cJSON *root = NULL; char file[PATH_MAX]; TdFilePtr pFile = NULL; @@ -97,7 +97,7 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); PRASE_MNODE_OVER: - if (content != NULL) free(content); + if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); @@ -118,7 +118,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) { int32_t len = 0; int32_t maxLen = 4096; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", deployed); @@ -139,7 +139,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) { taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); - free(content); + taosMemoryFree(content); char realfile[PATH_MAX]; snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP); diff --git a/source/dnode/mgmt/mnode/src/mmInt.c b/source/dnode/mgmt/mnode/src/mmInt.c index 1f60007be1..61afcb11d1 100644 --- a/source/dnode/mgmt/mnode/src/mmInt.c +++ b/source/dnode/mgmt/mnode/src/mmInt.c @@ -45,7 +45,8 @@ static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { SMsgCb msgCb = {0}; msgCb.pWrapper = pMgmt->pWrapper; - msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToReadQueue; + msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToQueryQueue; + msgCb.queueFps[READ_QUEUE] = mmPutMsgToReadQueue; msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue; msgCb.sendReqFp = dndSendReqToDnode; msgCb.sendMnodeReqFp = dndSendReqToMnode; @@ -177,7 +178,7 @@ int32_t mmDrop(SMgmtWrapper *pWrapper) { mmCloseImp(pMgmt); taosRemoveDir(pMgmt->path); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("mnode-mgmt is dropped"); return 0; } @@ -189,7 +190,7 @@ static void mmClose(SMgmtWrapper *pWrapper) { dInfo("mnode-mgmt start to cleanup"); mmCloseImp(pMgmt); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("mnode-mgmt is cleaned up"); } @@ -200,7 +201,7 @@ int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) { return -1; } - SMnodeMgmt *pMgmt = calloc(1, sizeof(SMnodeMgmt)); + SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt)); if (pMgmt == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -258,4 +259,4 @@ int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo SMonGrantInfo *pGrantInfo) { SMnodeMgmt *pMgmt = pWrapper->pMgmt; return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/mnode/src/mmMsg.c b/source/dnode/mgmt/mnode/src/mmMsg.c index 1cae2220ad..d04077baf8 100644 --- a/source/dnode/mgmt/mnode/src/mmMsg.c +++ b/source/dnode/mgmt/mnode/src/mmMsg.c @@ -156,9 +156,10 @@ void mmInitMsgHandles(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)mmProcessReadMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)mmProcessReadMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)mmProcessReadMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)mmProcessReadMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)mmProcessQueryMsg, MND_VGID); } diff --git a/source/dnode/mgmt/mnode/src/mmWorker.c b/source/dnode/mgmt/mnode/src/mmWorker.c index d6b150106d..27489b45d0 100644 --- a/source/dnode/mgmt/mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mnode/src/mmWorker.c @@ -44,6 +44,30 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { taosFreeQitem(pMsg); } +static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SMnodeMgmt *pMgmt = pInfo->ahandle; + + dTrace("msg:%p, will be processed in mnode queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + pMsg->pNode = pMgmt->pMnode; + code = mndProcessMsg(pMsg); + + if (pRpc->msgType & 1U) { + if (pRpc->handle == NULL) return; + if (code != 0) { + SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle}; + dndSendRsp(pMgmt->pWrapper, &rsp); + } + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + + static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SNodeMsg *pMsg) { dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); return taosWriteQitem(pWorker->queue, pMsg); @@ -61,6 +85,10 @@ int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); } +int32_t mmProcessQueryMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); +} + static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); if (pMsg == NULL) { @@ -90,8 +118,20 @@ int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc); } +int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->queryWorker, pRpc); +} + + int32_t mmStartWorker(SMnodeMgmt *pMgmt) { SSingleWorkerCfg cfg = {.minNum = 0, .maxNum = 1, .name = "mnode-read", .fp = (FItem)mmProcessQueue, .param = pMgmt}; + SSingleWorkerCfg queryCfg = {.minNum = 0, .maxNum = 1, .name = "mnode-query", .fp = (FItem)mmProcessQueryQueue, .param = pMgmt}; + + if (tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg) != 0) { + dError("failed to start mnode-query worker since %s", terrstr()); + return -1; + } if (tSingleWorkerInit(&pMgmt->readWorker, &cfg) != 0) { dError("failed to start mnode-read worker since %s", terrstr()); @@ -114,6 +154,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { void mmStopWorker(SMnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->readWorker); + tSingleWorkerCleanup(&pMgmt->queryWorker); tSingleWorkerCleanup(&pMgmt->writeWorker); tSingleWorkerCleanup(&pMgmt->syncWorker); dDebug("mnode workers are closed"); diff --git a/source/dnode/mgmt/qnode/src/qmInt.c b/source/dnode/mgmt/qnode/src/qmInt.c index 2bbfbd83f5..c8cb7258c3 100644 --- a/source/dnode/mgmt/qnode/src/qmInt.c +++ b/source/dnode/mgmt/qnode/src/qmInt.c @@ -76,7 +76,7 @@ int32_t qmDrop(SMgmtWrapper *pWrapper) { qmCloseImp(pMgmt); taosRemoveDir(pMgmt->path); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("qnode-mgmt is dropped"); return 0; } @@ -88,13 +88,13 @@ static void qmClose(SMgmtWrapper *pWrapper) { dInfo("qnode-mgmt start to cleanup"); qmCloseImp(pMgmt); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("qnode-mgmt is cleaned up"); } int32_t qmOpen(SMgmtWrapper *pWrapper) { dInfo("qnode-mgmt start to init"); - SQnodeMgmt *pMgmt = calloc(1, sizeof(SQnodeMgmt)); + SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt)); if (pMgmt == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/mgmt/snode/src/smInt.c b/source/dnode/mgmt/snode/src/smInt.c index b4d6576e57..351f7d656e 100644 --- a/source/dnode/mgmt/snode/src/smInt.c +++ b/source/dnode/mgmt/snode/src/smInt.c @@ -73,7 +73,7 @@ int32_t smDrop(SMgmtWrapper *pWrapper) { smCloseImp(pMgmt); taosRemoveDir(pMgmt->path); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("snode-mgmt is dropped"); return 0; } @@ -85,13 +85,13 @@ static void smClose(SMgmtWrapper *pWrapper) { dInfo("snode-mgmt start to cleanup"); smCloseImp(pMgmt); pWrapper->pMgmt = NULL; - free(pMgmt); + taosMemoryFree(pMgmt); dInfo("snode-mgmt is cleaned up"); } int32_t smOpen(SMgmtWrapper *pWrapper) { dInfo("snode-mgmt start to init"); - SSnodeMgmt *pMgmt = calloc(1, sizeof(SSnodeMgmt)); + SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt)); if (pMgmt == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/mgmt/snode/src/smWorker.c b/source/dnode/mgmt/snode/src/smWorker.c index 18ce71b8e7..5913713ff3 100644 --- a/source/dnode/mgmt/snode/src/smWorker.c +++ b/source/dnode/mgmt/snode/src/smWorker.c @@ -51,7 +51,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { } for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) { - SMultiWorker *pUniqueWorker = malloc(sizeof(SMultiWorker)); + SMultiWorker *pUniqueWorker = taosMemoryMalloc(sizeof(SMultiWorker)); if (pUniqueWorker == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -96,14 +96,15 @@ void smStopWorker(SSnodeMgmt *pMgmt) { static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) { SMsgHead *pHead = pMsg->pCont; - pHead->streamTaskId = htonl(pHead->streamTaskId); - return pHead->streamTaskId % SND_UNIQUE_THREAD_NUM; + pHead->vgId = htonl(pHead->vgId); + return pHead->vgId % SND_UNIQUE_THREAD_NUM; } static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) { - SStreamExecMsgHead *pHead = pMsg->pCont; - pHead->workerType = htonl(pHead->workerType); - return pHead->workerType; + /*SMsgHead *pHead = pMsg->pCont;*/ + /*pHead->workerType = htonl(pHead->workerType);*/ + /*return pHead->workerType;*/ + return 0; } int32_t smProcessMgmtMsg(SSnodeMgmt *pMgmt, SNodeMsg *pMsg) { diff --git a/source/dnode/mgmt/test/sut/inc/sut.h b/source/dnode/mgmt/test/sut/inc/sut.h index 304a370bcd..f7e2831160 100644 --- a/source/dnode/mgmt/test/sut/inc/sut.h +++ b/source/dnode/mgmt/test/sut/inc/sut.h @@ -98,7 +98,7 @@ class Testbase { #define CheckBinaryByte(b, len) \ { \ - char* bytes = (char*)calloc(1, len); \ + char* bytes = (char*)taosMemoryCalloc(1, len); \ for (int32_t i = 0; i < len - 1; ++i) { \ bytes[i] = b; \ } \ diff --git a/source/dnode/mgmt/test/sut/src/client.cpp b/source/dnode/mgmt/test/sut/src/client.cpp index f22bc9d276..a1165d5bc9 100644 --- a/source/dnode/mgmt/test/sut/src/client.cpp +++ b/source/dnode/mgmt/test/sut/src/client.cpp @@ -25,9 +25,9 @@ static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { void TestClient::SetRpcRsp(SRpcMsg* rsp) { if (this->pRsp) { - free(this->pRsp); + taosMemoryFree(this->pRsp); } - this->pRsp = (SRpcMsg*)calloc(1, sizeof(SRpcMsg)); + this->pRsp = (SRpcMsg*)taosMemoryCalloc(1, sizeof(SRpcMsg)); this->pRsp->msgType = rsp->msgType; this->pRsp->code = rsp->code; this->pRsp->pCont = rsp->pCont; diff --git a/source/dnode/mgmt/vnode/inc/vmInt.h b/source/dnode/mgmt/vnode/inc/vmInt.h index ccdb1ae257..197c606a0d 100644 --- a/source/dnode/mgmt/vnode/inc/vmInt.h +++ b/source/dnode/mgmt/vnode/inc/vmInt.h @@ -33,6 +33,7 @@ typedef struct SVnodesMgmt { SQWorkerPool fetchPool; SWWorkerPool syncPool; SWWorkerPool writePool; + SWWorkerPool mergePool; const char *path; SDnode *pDnode; SMgmtWrapper *pWrapper; @@ -63,6 +64,7 @@ typedef struct { STaosQueue *pApplyQ; STaosQueue *pQueryQ; STaosQueue *pFetchQ; + STaosQueue *pMergeQ; SMgmtWrapper *pWrapper; } SVnodeObj; @@ -110,10 +112,11 @@ int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); +int32_t vmProcessMergeMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg); #ifdef __cplusplus } #endif -#endif /*_TD_DND_VNODES_INT_H_*/ \ No newline at end of file +#endif /*_TD_DND_VNODES_INT_H_*/ diff --git a/source/dnode/mgmt/vnode/src/vmFile.c b/source/dnode/mgmt/vnode/src/vmFile.c index 0fe868dfa4..d4f906b482 100644 --- a/source/dnode/mgmt/vnode/src/vmFile.c +++ b/source/dnode/mgmt/vnode/src/vmFile.c @@ -21,7 +21,7 @@ SVnodeObj **vmGetVnodesFromHash(SVnodesMgmt *pMgmt, int32_t *numOfVnodes) { int32_t num = 0; int32_t size = taosHashGetSize(pMgmt->hash); - SVnodeObj **pVnodes = calloc(size, sizeof(SVnodeObj *)); + SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *)); void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { @@ -48,7 +48,7 @@ int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *n int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; int32_t len = 0; int32_t maxLen = 30000; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); cJSON *root = NULL; FILE *fp = NULL; char file[PATH_MAX]; @@ -85,7 +85,7 @@ int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *n int32_t vnodesNum = cJSON_GetArraySize(vnodes); if (vnodesNum > 0) { - pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); + pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg)); if (pCfgs == NULL) { dError("failed to read %s since out of memory", file); goto PRASE_VNODE_OVER; @@ -140,7 +140,7 @@ int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *n dInfo("succcessed to read file %s", file); PRASE_VNODE_OVER: - if (content != NULL) free(content); + if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); @@ -166,7 +166,7 @@ int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) { int32_t len = 0; int32_t maxLen = 65536; - char *content = calloc(1, maxLen + 1); + char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); @@ -190,7 +190,7 @@ int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) { taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); - free(content); + taosMemoryFree(content); terrno = 0; for (int32_t i = 0; i < numOfVnodes; ++i) { @@ -199,7 +199,7 @@ int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) { } if (pVnodes != NULL) { - free(pVnodes); + taosMemoryFree(pVnodes); } dDebug("successed to write %s", realfile); diff --git a/source/dnode/mgmt/vnode/src/vmInt.c b/source/dnode/mgmt/vnode/src/vmInt.c index 71f8f23b24..464e789e46 100644 --- a/source/dnode/mgmt/vnode/src/vmInt.c +++ b/source/dnode/mgmt/vnode/src/vmInt.c @@ -46,7 +46,7 @@ void vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { } int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { - SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); + SVnodeObj *pVnode = taosMemoryCalloc(1, sizeof(SVnodeObj)); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -107,9 +107,9 @@ void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { vnodeDestroy(pVnode->path); } - free(pVnode->path); - free(pVnode->db); - free(pVnode); + taosMemoryFree(pVnode->path); + taosMemoryFree(pVnode->db); + taosMemoryFree(pVnode); } static void *vmOpenVnodeFunc(void *param) { @@ -183,11 +183,11 @@ static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) { #endif int32_t vnodesPerThread = numOfVnodes / threadNum + 1; - SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); + SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread)); for (int32_t t = 0; t < threadNum; ++t) { threads[t].threadIndex = t; threads[t].pMgmt = pMgmt; - threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); + threads[t].pCfgs = taosMemoryCalloc(vnodesPerThread, sizeof(SWrapperCfg)); } for (int32_t v = 0; v < numOfVnodes; ++v) { @@ -217,10 +217,10 @@ static int32_t vmOpenVnodes(SVnodesMgmt *pMgmt) { if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { taosThreadJoin(pThread->thread, NULL); } - free(pThread->pCfgs); + taosMemoryFree(pThread->pCfgs); } - free(threads); - free(pCfgs); + taosMemoryFree(threads); + taosMemoryFree(pCfgs); if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes); @@ -242,7 +242,7 @@ static void vmCloseVnodes(SVnodesMgmt *pMgmt) { } if (pVnodes != NULL) { - free(pVnodes); + taosMemoryFree(pVnodes); } if (pMgmt->hash != NULL) { @@ -262,14 +262,14 @@ static void vmCleanup(SMgmtWrapper *pWrapper) { vmStopWorker(pMgmt); vnodeCleanup(); // walCleanUp(); - free(pMgmt); + taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; dInfo("vnode-mgmt is cleaned up"); } static int32_t vmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; - SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt)); + SVnodesMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodesMgmt)); int32_t code = -1; dInfo("vnode-mgmt start to init"); diff --git a/source/dnode/mgmt/vnode/src/vmMsg.c b/source/dnode/mgmt/vnode/src/vmMsg.c index 29696b4079..97d829571f 100644 --- a/source/dnode/mgmt/vnode/src/vmMsg.c +++ b/source/dnode/mgmt/vnode/src/vmMsg.c @@ -280,6 +280,8 @@ void vmInitMsgHandles(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg, VND_VGID); diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c index 12e15e5adf..e97d6e7f11 100644 --- a/source/dnode/mgmt/vnode/src/vmWorker.c +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -117,7 +117,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO if (pRsp != NULL) { pRsp->ahandle = pRpc->ahandle; dndSendRsp(pVnode->pWrapper, pRsp); - free(pRsp); + taosMemoryFree(pRsp); } else { if (code != 0 && terrno != 0) code = terrno; vmSendRsp(pVnode->pWrapper, pMsg, code); @@ -165,8 +165,8 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp int32_t code = -1; SMsgHead *pHead = pRpc->pCont; - pHead->contLen = htonl(pHead->contLen); - pHead->vgId = htonl(pHead->vgId); + pHead->contLen = ntohl(pHead->contLen); + pHead->vgId = ntohl(pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { @@ -191,6 +191,10 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp dTrace("msg:%p, will be written into vnode-sync queue", pMsg); code = taosWriteQitem(pVnode->pSyncQ, pMsg); break; + case MERGE_QUEUE: + dTrace("msg:%p, will be written into vnode-merge queue", pMsg); + code = taosWriteQitem(pVnode->pMergeQ, pMsg); + break; default: terrno = TSDB_CODE_INVALID_PARA; break; @@ -208,6 +212,8 @@ int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNode int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); } +int32_t vmProcessMergeMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return vmPutNodeMsgToQueue(pMgmt, pMsg, MERGE_QUEUE); } + int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->mgmtWorker; dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name); @@ -239,6 +245,10 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT dTrace("msg:%p, will be put into vnode-apply queue", pMsg); code = taosWriteQitem(pVnode->pApplyQ, pMsg); break; + case MERGE_QUEUE: + dTrace("msg:%p, will be put into vnode-merge queue", pMsg); + code = taosWriteQitem(pVnode->pMergeQ, pMsg); + break; default: terrno = TSDB_CODE_INVALID_PARA; break; @@ -260,6 +270,10 @@ int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { return vmPutRpcMsgToQueue(pWrapper, pRpc, APPLY_QUEUE); } +int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE); +} + int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { int32_t size = -1; SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId); @@ -280,6 +294,9 @@ int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { case APPLY_QUEUE: size = taosQueueSize(pVnode->pApplyQ); break; + case MERGE_QUEUE: + size = taosQueueSize(pVnode->pMergeQ); + break; default: break; } @@ -291,12 +308,13 @@ int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); + pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeMsg); pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || - pVnode->pQueryQ == NULL) { + pVnode->pQueryQ == NULL || pVnode->pMergeQ == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -310,12 +328,14 @@ void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + tWWorkerFreeQueue(&pMgmt->mergePool, pVnode->pMergeQ); tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); pVnode->pWriteQ = NULL; pVnode->pApplyQ = NULL; pVnode->pSyncQ = NULL; pVnode->pFetchQ = NULL; pVnode->pQueryQ = NULL; + pVnode->pMergeQ = NULL; dDebug("vgId:%d, vnode queue is freed", pVnode->vgId); } @@ -326,6 +346,7 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { int32_t maxQueryThreads = minQueryThreads; int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); + int32_t maxMergeThreads = 1; SQWorkerPool *pQPool = &pMgmt->queryPool; pQPool->name = "vnode-query"; @@ -349,6 +370,11 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { pWPool->max = maxSyncThreads; if (tWWorkerInit(pWPool) != 0) return -1; + pWPool = &pMgmt->mergePool; + pWPool->name = "vnode-merge"; + pWPool->max = maxMergeThreads; + if (tWWorkerInit(pWPool) != 0) return -1; + SSingleWorkerCfg cfg = { .minNum = 1, .maxNum = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt}; if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) { @@ -366,5 +392,6 @@ void vmStopWorker(SVnodesMgmt *pMgmt) { tQWorkerCleanup(&pMgmt->queryPool); tWWorkerCleanup(&pMgmt->writePool); tWWorkerCleanup(&pMgmt->syncPool); + tWWorkerCleanup(&pMgmt->mergePool); dDebug("vnode workers are closed"); } diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index dd2caf1f7f..e91c851be7 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - mnode scheduler sdb wal transport cjson sync monitor parser + mnode scheduler sdb wal transport cjson sync monitor executor qworker stream parser ) if(${BUILD_TEST}) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 0d65dacb20..caf5172596 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -26,6 +26,7 @@ #include "tlog.h" #include "tmsg.h" #include "trpc.h" +#include "tstream.h" #include "ttimer.h" #include "mnode.h" @@ -441,7 +442,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) { if (pConsumerEp) { - tfree(pConsumerEp->qmsg); + taosMemoryFreeClear(pConsumerEp->qmsg); } } @@ -510,7 +511,7 @@ typedef struct { } SMqSubscribeObj; static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { - SMqSubscribeObj* pSub = calloc(1, sizeof(SMqSubscribeObj)); + SMqSubscribeObj* pSub = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); if (pSub == NULL) { return NULL; } @@ -537,10 +538,10 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { return pSub; _err: - tfree(pSub->consumers); - tfree(pSub->lostConsumers); - tfree(pSub->unassignedVg); - tfree(pSub); + taosMemoryFreeClear(pSub->consumers); + taosMemoryFreeClear(pSub->lostConsumers); + taosMemoryFreeClear(pSub->unassignedVg); + taosMemoryFreeClear(pSub); return NULL; } @@ -632,18 +633,19 @@ static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { } typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - int64_t uid; - int64_t dbUid; - int32_t version; - SRWLatch lock; - int32_t sqlLen; - char* sql; - char* logicalPlan; - char* physicalPlan; + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; + int64_t dbUid; + int32_t version; + SRWLatch lock; + int32_t sqlLen; + char* sql; + char* logicalPlan; + char* physicalPlan; + SSchemaWrapper schema; } SMqTopicObj; typedef struct { @@ -729,13 +731,15 @@ typedef struct { int32_t vgNum; SRWLatch lock; int8_t status; + int8_t sourceType; + int8_t sinkType; // int32_t sqlLen; - int32_t sinkVgId; // 0 for automatic - char* sql; - char* logicalPlan; - char* physicalPlan; - SArray* tasks; // SArray> - SArray* ColAlias; + int32_t sinkVgId; // 0 for automatic + char* sql; + char* logicalPlan; + char* physicalPlan; + SArray* tasks; // SArray> + SSchemaWrapper outputSchema; } SStreamObj; int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 20e85973be..f89e9d8fe0 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -59,6 +59,8 @@ typedef struct SMnodeLoad { int64_t compStorage; } SMnodeLoad; +typedef struct SQWorkerMgmt SQHandle; + typedef struct { const char *name; MndInitFp initFp; @@ -112,6 +114,7 @@ typedef struct SMnode { SSdb *pSdb; SMgmtWrapper *pWrapper; SArray *pSteps; + SQHandle *pQuery; SShowMgmt showMgmt; SProfileMgmt profileMgmt; STelemMgmt telemMgmt; @@ -119,7 +122,7 @@ typedef struct SMnode { SHashObj *infosMeta; SGrantInfo grant; MndMsgFp msgFp[TDMT_MAX]; - SMsgCb msgCb; + SMsgCb msgCb; } SMnode; void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); diff --git a/source/dnode/mnode/impl/inc/mndQuery.h b/source/dnode/mnode/impl/inc/mndQuery.h new file mode 100644 index 0000000000..7fab80de77 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndQuery.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_QUERY_H_ +#define _TD_MND_QUERY_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitQuery(SMnode *pMnode); +void mndCleanupQuery(SMnode *pMnode); + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_QUERY_H_*/ diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index a6e6d57345..68c8c6d127 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -158,7 +158,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { ACCT_DECODE_OVER: if (terrno != 0) { mError("acct:%s, failed to decode from raw:%p since %s", pAcct->acct, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index d9a0df555c..c4868d9802 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -126,7 +126,7 @@ static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw) { BNODE_DECODE_OVER: if (terrno != 0) { mError("bnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -191,7 +191,7 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -206,7 +206,7 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -218,7 +218,7 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -233,7 +233,7 @@ static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -341,7 +341,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -356,7 +356,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index ce47080645..dde7e1fe8f 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -116,7 +116,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { CLUSTER_DECODE_OVER: if (terrno != 0) { mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index d86665b832..8eceea3d06 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -54,7 +54,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) { - SMqConsumerObj *pConsumer = calloc(1, sizeof(SMqConsumerObj)); + SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -79,7 +79,7 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) goto CM_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto CM_ENCODE_OVER; void *abuf = buf; @@ -94,7 +94,7 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_SUCCESS; CM_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != 0) { mError("consumer:%" PRId64 ", failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -126,7 +126,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t len; SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); - buf = malloc(len); + buf = taosMemoryMalloc(len); if (buf == NULL) goto CM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); @@ -138,10 +138,10 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; CM_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("consumer:%" PRId64 ", failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 183e142624..05c7d79a1a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -193,7 +193,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { DB_DECODE_OVER: if (terrno != 0) { mError("db:%s, failed to decode from raw:%p since %s", pDb->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -370,7 +370,7 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -401,7 +401,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -483,7 +483,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate code = 0; CREATE_DB_OVER: - free(pVgroups); + taosMemoryFree(pVgroups); mndTransDrop(pTrans); return code; } @@ -622,7 +622,7 @@ static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_DND_ALTER_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -802,7 +802,7 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -845,7 +845,7 @@ static int32_t mndBuildDropDbRsp(SDbObj *pDb, int32_t *pRspLen, void **ppRsp, bo if (useRpcMalloc) { pRsp = rpcMallocCont(rspLen); } else { - pRsp = malloc(rspLen); + pRsp = taosMemoryMalloc(rspLen); } if (pRsp == NULL) { @@ -1034,7 +1034,9 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { usedbRsp.vgVersion = usedbReq.vgVersion; code = 0; } - usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + + // no jump, need to construct rsp } else { pDb = mndAcquireDb(pMnode, usedbReq.db); if (pDb == NULL) { @@ -1156,7 +1158,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, } int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); - void *pRsp = malloc(rspLen); + void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeSUseDbBatchRsp(&batchUseRsp); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6fa926d548..24f2a5df22 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -17,7 +17,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { int32_t sz = 0; - int32_t outputNameSz = 0; + /*int32_t outputNameSz = 0;*/ if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1; if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; @@ -45,6 +45,9 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { } } + if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; + +#if 0 if (pObj->ColAlias != NULL) { outputNameSz = taosArrayGetSize(pObj->ColAlias); } @@ -53,6 +56,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { char *name = taosArrayGetP(pObj->ColAlias, i); if (tEncodeCStr(pEncoder, name) < 0) return -1; } +#endif return pEncoder->pos; } @@ -85,6 +89,9 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { taosArrayPush(pObj->tasks, pArray); } } + + if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; +#if 0 int32_t outputNameSz; if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1; if (outputNameSz != 0) { @@ -98,5 +105,6 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1; taosArrayPush(pObj->ColAlias, &name); } +#endif return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 3e47f3a9e8..314e70db9b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -164,7 +164,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { DNODE_DECODE_OVER: if (terrno != 0) { mError("dnode:%d, failed to decode from raw:%p since %s", pDnode->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -277,8 +277,8 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } - if ((0 != strcasecmp(pCfg->timezone, tsTimezone)) && (pMnode->checkTime != pCfg->checkTime)) { - mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, tsTimezone, + if ((0 != strcasecmp(pCfg->timezone, tsTimezoneStr)) && (pMnode->checkTime != pCfg->checkTime)) { + mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, tsTimezoneStr, pCfg->checkTime, pMnode->checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } @@ -677,7 +677,7 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, char *data, i totalRows++; cfgOpts[totalRows] = "timezone"; - snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsTimezone); + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsTimezoneStr); totalRows++; cfgOpts[totalRows] = "locale"; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b43d887dd7..be4198f0d6 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -127,8 +127,8 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, FUNC_DECODE_OVER) - pFunc->pComment = calloc(1, pFunc->commentSize); - pFunc->pCode = calloc(1, pFunc->codeSize); + pFunc->pComment = taosMemoryCalloc(1, pFunc->commentSize); + pFunc->pCode = taosMemoryCalloc(1, pFunc->codeSize); if (pFunc->pComment == NULL || pFunc->pCode == NULL) { goto FUNC_DECODE_OVER; } @@ -142,7 +142,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { FUNC_DECODE_OVER: if (terrno != 0) { mError("func:%s, failed to decode from raw:%p since %s", pFunc->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -157,8 +157,8 @@ static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc) { static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { mTrace("func:%s, perform delete action, row:%p", pFunc->name, pFunc); - tfree(pFunc->pCode); - tfree(pFunc->pComment); + taosMemoryFreeClear(pFunc->pCode); + taosMemoryFreeClear(pFunc->pComment); return 0; } @@ -196,8 +196,8 @@ static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCr func.signature = pCreate->signature; func.commentSize = pCreate->commentSize; func.codeSize = pCreate->codeSize; - func.pComment = malloc(func.commentSize); - func.pCode = malloc(func.codeSize); + func.pComment = taosMemoryMalloc(func.commentSize); + func.pCode = taosMemoryMalloc(func.codeSize); if (func.pCode == NULL || func.pCode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto CREATE_FUNC_OVER; @@ -228,8 +228,8 @@ static int32_t mndCreateFunc(SMnode *pMnode, SNodeMsg *pReq, SCreateFuncReq *pCr code = 0; CREATE_FUNC_OVER: - free(func.pCode); - free(func.pComment); + taosMemoryFree(func.pCode); + taosMemoryFree(func.pComment); mndTransDrop(pTrans); return code; } diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 964c4ab424..c9b69bc161 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -19,6 +19,7 @@ #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +//!!!! Note: only APPEND columns in below tables, NO insert !!!! static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, @@ -152,7 +153,7 @@ static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema //connection/application/ int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) { - SSchema *schema = calloc(colNum, sizeof(SSchema)); + SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); if (NULL == schema) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -210,7 +211,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char * *pRsp = *meta; - pRsp->pSchemas = calloc(meta->numOfColumns, sizeof(SSchema)); + pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema)); if (pRsp->pSchemas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pRsp->pSchemas = NULL; @@ -241,7 +242,7 @@ void mndCleanupInfos(SMnode *pMnode) { while (pIter) { STableMetaRsp *meta = (STableMetaRsp *)pIter; - tfree(meta->pSchemas); + taosMemoryFreeClear(meta->pSchemas); pIter = taosHashIterate(pMnode->infosMeta, pIter); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 703f36bb33..c408fea636 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -180,7 +180,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { MNODE_DECODE_OVER: if (terrno != 0) { mError("mnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -313,7 +313,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno createReq.dnodeId = pMObj->id; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); action.epSet = mndGetDnodeEpset(pMObj->pDnode); @@ -323,7 +323,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; @@ -338,7 +338,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno createReq.dnodeId = pObj->id; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); action.epSet = mndGetDnodeEpset(pDnode); @@ -347,7 +347,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno action.msgType = TDMT_DND_CREATE_MNODE; action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } @@ -483,7 +483,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode alterReq.dnodeId = pMObj->id; int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); action.epSet = mndGetDnodeEpset(pMObj->pDnode); @@ -493,7 +493,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; @@ -510,7 +510,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode SDDropMnodeReq dropReq = {0}; dropReq.dnodeId = pObj->id; int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); action.epSet = mndGetDnodeEpset(pDnode); @@ -519,7 +519,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.msgType = TDMT_DND_DROP_MNODE; action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 38157cf220..22dad48772 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -59,7 +59,7 @@ SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size); if (pRaw == NULL) goto OFFSET_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto OFFSET_ENCODE_OVER; void *abuf = buf; @@ -74,7 +74,7 @@ SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { terrno = TSDB_CODE_SUCCESS; OFFSET_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -107,7 +107,7 @@ SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER); - buf = malloc(tlen + 1); + buf = taosMemoryMalloc(tlen + 1); if (buf == NULL) goto OFFSET_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER); @@ -119,10 +119,10 @@ SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; OFFSET_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 7ca1984457..bf378a4d43 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -130,7 +130,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, } static void mndFreeConn(SConnObj *pConn) { - tfree(pConn->pQueries); + taosMemoryFreeClear(pConn->pQueries); mTrace("conn:%d, is destroyed, data:%p", pConn->id, pConn); } @@ -260,7 +260,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { if (numOfQueries > 0) { if (pConn->pQueries == NULL) { - pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); + pConn->pQueries = taosMemoryCalloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); } pConn->numOfQueries = TMIN(QUERY_SAVE_SIZE, numOfQueries); @@ -276,7 +276,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { #if 0 - SClientHbRsp* pRsp = malloc(sizeof(SClientHbRsp)); + SClientHbRsp* pRsp = taosMemoryMalloc(sizeof(SClientHbRsp)); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -292,7 +292,7 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { SHashObj* pObj = pReq->info; SKv* pKv = taosHashGet(pObj, "mq-tmp", strlen("mq-tmp") + 1); if (pKv == NULL) { - free(pRsp); + taosMemoryFree(pRsp); return NULL; } SMqHbMsg mqHb; @@ -325,7 +325,7 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { taosArrayPush(batchRsp.batchRsps, &innerBatchRsp); } int32_t tlen = taosEncodeSMqHbBatchRsp(NULL, &batchRsp); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { //TODO return NULL; @@ -402,7 +402,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { SClientHbRsp *pRsp = mndMqHbBuildRsp(pMnode, pHbReq); if (pRsp != NULL) { taosArrayPush(batchRsp.rsps, pRsp); - free(pRsp); + taosMemoryFree(pRsp); } } } @@ -418,7 +418,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) { int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info) : 0; for (int32_t n = 0; n < kvNum; ++n) { SKv *kv = taosArrayGet(rsp->info, n); - tfree(kv->value); + taosMemoryFreeClear(kv->value); } taosArrayDestroy(rsp->info); } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 3c66160c91..4b19a26bc4 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -128,7 +128,7 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) { QNODE_DECODE_OVER: if (terrno != 0) { mError("qnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -193,7 +193,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -208,7 +208,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -220,7 +220,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -235,7 +235,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -343,7 +343,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -358,7 +358,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -474,7 +474,7 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { } int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); - void *pRsp = malloc(rspLen); + void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto QNODE_LIST_OVER; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c new file mode 100644 index 0000000000..e93a0d9b17 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mndQuery.h" +#include "mndMnode.h" +#include "executor.h" +#include "qworker.h" + +int32_t mndProcessQueryMsg(SNodeMsg *pReq) { + mTrace("message in query queue is processing"); + SMnode *pMnode = pReq->pNode; + SReadHandle handle = {0}; + + switch (pReq->rpcMsg.msgType) { + case TDMT_VND_QUERY: + return qWorkerProcessQueryMsg(&handle, pMnode->pQuery, &pReq->rpcMsg); + case TDMT_VND_QUERY_CONTINUE: + return qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, &pReq->rpcMsg); + default: + mError("unknown msg type:%d in query queue", pReq->rpcMsg.msgType); + return TSDB_CODE_VND_APP_ERROR; + } +} + +int32_t mndProcessFetchMsg(SNodeMsg *pReq) { + mTrace("message in fetch queue is processing"); + SMnode *pMnode = pReq->pNode; + + switch (pReq->rpcMsg.msgType) { + case TDMT_VND_FETCH: + return qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, &pReq->rpcMsg); + case TDMT_VND_DROP_TASK: + return qWorkerProcessDropMsg(pMnode, pMnode->pQuery, &pReq->rpcMsg); + case TDMT_VND_QUERY_HEARTBEAT: + return qWorkerProcessHbMsg(pMnode, pMnode->pQuery, &pReq->rpcMsg); + default: + mError("unknown msg type:%d in fetch queue", pReq->rpcMsg.msgType); + return TSDB_CODE_VND_APP_ERROR; + } +} + +int32_t mndInitQuery(SMnode *pMnode) { + int32_t code = qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb); + if (code) { + return code; + } + + mndSetMsgHandle(pMnode, TDMT_VND_QUERY, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_VND_QUERY_CONTINUE, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_VND_FETCH, mndProcessFetchMsg); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_TASK, mndProcessFetchMsg); + mndSetMsgHandle(pMnode, TDMT_VND_QUERY_HEARTBEAT, mndProcessFetchMsg); + + return 0; +} + +void mndCleanupQuery(SMnode *pMnode) { qWorkerDestroy((void **)&pMnode->pQuery); } + diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 870cbba979..a4dfd293de 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -41,12 +41,12 @@ int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet int32_t size = encoder.pos; int32_t tlen = sizeof(SMsgHead) + size; tCoderClear(&encoder); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - ((SMsgHead*)buf)->streamTaskId = htonl(nodeId); + ((SMsgHead*)buf)->vgId = htonl(nodeId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, size, TD_ENCODER); tEncodeSStreamTask(&encoder, pTask); @@ -58,7 +58,7 @@ int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet action.contLen = tlen; action.msgType = type; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } return 0; @@ -66,10 +66,13 @@ int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t msgLen; - plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); + pTask->nodeId = pVgroup->vgId; + pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.epSet = pTask->epSet; + + if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } @@ -86,10 +89,14 @@ SSnodeObj* mndSchedFetchSnode(SMnode* pMnode) { int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, const SSnodeObj* pSnode) { int32_t msgLen; - plan->execNode.nodeId = pSnode->id; - plan->execNode.epSet = mndAcquireEpFromSnode(pMnode, pSnode); - if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) { + pTask->nodeId = 0; + pTask->epSet = mndAcquireEpFromSnode(pMnode, pSnode); + + plan->execNode.nodeId = 0; + plan->execNode.epSet = pTask->epSet; + + if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } @@ -97,9 +104,23 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, return 0; } +SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { + void* pIter = NULL; + SVgObj* pVgroup = NULL; + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != dbUid) { + sdbRelease(pMnode->pSdb, pVgroup); + continue; + } + return pVgroup; + } + return pVgroup; +} + int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; - SVgObj* pVgroup = NULL; SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); if (pPlan == NULL) { terrno = TSDB_CODE_QRY_INVALID_INPUT; @@ -108,74 +129,144 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { ASSERT(pStream->vgNum == 0); int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); - pStream->tasks = taosArrayInit(totLevel, sizeof(SArray)); - int32_t lastUsedVgId = 0; + ASSERT(totLevel <= 2); + pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); for (int32_t level = 0; level < totLevel; level++) { - SArray* taskOneLevel = taosArrayInit(0, sizeof(SStreamTask)); + SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level); - int32_t opNum = LIST_LENGTH(inner->pNodeList); - ASSERT(opNum == 1); + ASSERT(LIST_LENGTH(inner->pNodeList) == 1); SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); - if (level == 0) { + + // if (level == totLevel - 1 /* or no snode */) { + if (level == totLevel - 1) { + // last level, source, must assign to vnode + // must be scan type ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); + + // replicate task to each vnode void* pIter = NULL; while (1) { + SVgObj* pVgroup; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); if (pIter == NULL) break; if (pVgroup->dbUid != pStream->dbUid) { sdbRelease(pSdb, pVgroup); continue; } + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + // source part + pTask->sourceType = TASK_SOURCE__SCAN; - lastUsedVgId = pVgroup->vgId; - pStream->vgNum++; + // sink part + if (level == 0) { + // only for inplace + pTask->sinkType = TASK_SINK__SHOW; + pTask->showSink.reserved = 0; + } else { + pTask->sinkType = TASK_SINK__NONE; + } - SStreamTask* pTask = streamTaskNew(pStream->uid); - pTask->level = level; - pTask->sourceType = 1; - pTask->sinkType = level == totLevel - 1 ? 1 : 0; - pTask->parallelizable = 1; + // dispatch part + if (level == 0) { + pTask->dispatchType = TASK_DISPATCH__NONE; + // if inplace sink, no dispatcher + // if fixed ep, add fixed ep dispatcher + // if shuffle, add shuffle dispatcher + } else { + // add fixed ep dispatcher + int32_t lastLevel = level - 1; + ASSERT(lastLevel == 0); + SArray* pArray = taosArrayGetP(pStream->tasks, lastLevel); + // one merge only + ASSERT(taosArrayGetSize(pArray) == 1); + SStreamTask* lastLevelTask = taosArrayGetP(pArray, lastLevel); + pTask->dispatchMsgType = TDMT_VND_TASK_MERGE_EXEC; + pTask->dispatchType = TASK_DISPATCH__FIXED; + + pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; + pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; + } + + // exec part + pTask->execType = TASK_EXEC__PIPE; + pTask->exec.parallelizable = 1; if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; } - taosArrayPush(taskOneLevel, pTask); + sdbRelease(pSdb, pVgroup); + taosArrayPush(taskOneLevel, &pTask); } } else { - SStreamTask* pTask = streamTaskNew(pStream->uid); - pTask->level = level; - pTask->sourceType = 0; - pTask->sinkType = level == totLevel - 1 ? 1 : 0; - pTask->parallelizable = plan->subplanType == SUBPLAN_TYPE_SCAN; - pTask->nextOpDst = STREAM_NEXT_OP_DST__VND; + // merge plan - SSnodeObj* pSnode = mndSchedFetchSnode(pMnode); - if (pSnode == NULL || tsStreamSchedV) { - ASSERT(lastUsedVgId != 0); - SVgObj* pVg = mndAcquireVgroup(pMnode, lastUsedVgId); - if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVg) < 0) { - sdbRelease(pSdb, pVg); - qDestroyQueryPlan(pPlan); - return -1; - } - sdbRelease(pSdb, pVg); - } else { - if (mndAssignTaskToSnode(pMnode, pTrans, pTask, plan, pSnode) < 0) { - sdbRelease(pSdb, pSnode); - qDestroyQueryPlan(pPlan); - return -1; - } + // TODO if has snode, assign to snode + + // else, assign to vnode + ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + + // source part, currently only support multi source + pTask->sourceType = TASK_SOURCE__PIPE; + + // sink part + pTask->sinkType = TASK_SINK__SHOW; + /*pTask->sinkType = TASK_SINK__NONE;*/ + + // dispatch part + pTask->dispatchType = TASK_DISPATCH__SHUFFLE; + pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC; + + // exec part + pTask->execType = TASK_EXEC__MERGE; + pTask->exec.parallelizable = 0; + SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->dbUid); + ASSERT(pVgroup); + if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; } - sdbRelease(pMnode->pSdb, pSnode); - - taosArrayPush(taskOneLevel, pTask); + sdbRelease(pSdb, pVgroup); + taosArrayPush(taskOneLevel, &pTask); } - taosArrayPush(pStream->tasks, taskOneLevel); + + taosArrayPush(pStream->tasks, &taskOneLevel); } + + if (totLevel == 2) { + void* pIter = NULL; + while (1) { + SVgObj* pVgroup; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pStream->dbUid) { + sdbRelease(pSdb, pVgroup); + continue; + } + SStreamTask* pTask = tNewSStreamTask(pStream->uid); + + // source part + pTask->sourceType = TASK_SOURCE__MERGE; + + // sink part + pTask->sinkType = TASK_SINK__SHOW; + + // dispatch part + pTask->dispatchType = TASK_DISPATCH__NONE; + + // exec part + pTask->execType = TASK_EXEC__NONE; + pTask->exec.parallelizable = 0; + } + } + + // free memory qDestroyQueryPlan(pPlan); + return 0; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index eb12347e64..dff918f135 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -146,7 +146,7 @@ static int32_t mndProcessShowReq(SNodeMsg *pReq) { } showRsp.showId = pShow->id; - showRsp.tableMeta.pSchemas = calloc(TSDB_MAX_COLUMNS, sizeof(SSchema)); + showRsp.tableMeta.pSchemas = taosMemoryCalloc(TSDB_MAX_COLUMNS, sizeof(SSchema)); if (showRsp.tableMeta.pSchemas == NULL) { mndReleaseShowObj(pShow, true); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -357,6 +357,13 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { // if free flag is set, client wants to clean the resources if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { rowsRead = (*retrieveFp)(pReq, (SShowObj*) pShow, pRsp->data, rowsToRead); + if (rowsRead < 0) { + terrno = rowsRead; + rpcFreeCont(pRsp); + mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); + mndReleaseShowObj((SShowObj*) pShow, true); + return -1; + } } mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 4e3ab8eae4..146975aa38 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -160,25 +160,25 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pSma->astLen, _OVER) if (pSma->exprLen > 0) { - pSma->expr = calloc(pSma->exprLen, 1); + pSma->expr = taosMemoryCalloc(pSma->exprLen, 1); if (pSma->expr == NULL) goto _OVER; SDB_GET_BINARY(pRaw, dataPos, pSma->expr, pSma->exprLen, _OVER) } if (pSma->tagsFilterLen > 0) { - pSma->tagsFilter = calloc(pSma->tagsFilterLen, 1); + pSma->tagsFilter = taosMemoryCalloc(pSma->tagsFilterLen, 1); if (pSma->tagsFilter == NULL) goto _OVER; SDB_GET_BINARY(pRaw, dataPos, pSma->tagsFilter, pSma->tagsFilterLen, _OVER) } if (pSma->sqlLen > 0) { - pSma->sql = calloc(pSma->sqlLen, 1); + pSma->sql = taosMemoryCalloc(pSma->sqlLen, 1); if (pSma->sql == NULL) goto _OVER; SDB_GET_BINARY(pRaw, dataPos, pSma->sql, pSma->sqlLen, _OVER) } if (pSma->astLen > 0) { - pSma->ast = calloc(pSma->astLen, 1); + pSma->ast = taosMemoryCalloc(pSma->astLen, 1); if (pSma->ast == NULL) goto _OVER; SDB_GET_BINARY(pRaw, dataPos, pSma->ast, pSma->astLen, _OVER) } @@ -189,9 +189,9 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { _OVER: if (terrno != 0) { mError("sma:%s, failed to decode from raw:%p since %s", pSma->name, pRaw, terrstr()); - tfree(pSma->expr); - tfree(pSma->tagsFilter); - tfree(pRow); + taosMemoryFreeClear(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pRow); return NULL; } @@ -206,8 +206,8 @@ static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma) { static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSma) { mTrace("sma:%s, perform delete action, row:%p", pSma->name, pSma); - tfree(pSma->tagsFilter); - tfree(pSma->expr); + taosMemoryFreeClear(pSma->tagsFilter); + taosMemoryFreeClear(pSma->expr); return 0; } @@ -261,7 +261,7 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm req.tSma.tagsFilter = pSma->tagsFilter; int32_t contLen = tSerializeSVCreateTSmaReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = malloc(contLen); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -287,7 +287,7 @@ static void *mndBuildVDropSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSma, tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); int32_t contLen = tSerializeSVDropTSmaReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = malloc(contLen); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -349,7 +349,7 @@ static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_CREATE_SMA; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -382,25 +382,25 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre smaObj.astLen = pCreate->astLen; if (smaObj.exprLen > 0) { - smaObj.expr = malloc(smaObj.exprLen); + smaObj.expr = taosMemoryMalloc(smaObj.exprLen); if (smaObj.expr == NULL) goto _OVER; memcpy(smaObj.expr, pCreate->expr, smaObj.exprLen); } if (smaObj.tagsFilterLen > 0) { - smaObj.tagsFilter = malloc(smaObj.tagsFilterLen); + smaObj.tagsFilter = taosMemoryMalloc(smaObj.tagsFilterLen); if (smaObj.tagsFilter == NULL) goto _OVER; memcpy(smaObj.tagsFilter, pCreate->tagsFilter, smaObj.tagsFilterLen); } if (smaObj.sqlLen > 0) { - smaObj.sql = malloc(smaObj.sqlLen); + smaObj.sql = taosMemoryMalloc(smaObj.sqlLen); if (smaObj.sql == NULL) goto _OVER; memcpy(smaObj.sql, pCreate->sql, smaObj.sqlLen); } if (smaObj.astLen > 0) { - smaObj.ast = malloc(smaObj.astLen); + smaObj.ast = taosMemoryMalloc(smaObj.astLen); if (smaObj.ast == NULL) goto _OVER; memcpy(smaObj.ast, pCreate->ast, smaObj.astLen); } @@ -596,7 +596,7 @@ static int32_t mndSetDropSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * action.msgType = TDMT_VND_DROP_SMA; action.acceptableCode = TSDB_CODE_VND_SMA_NOT_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index f01b143fbc..5e0d9fae9a 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -135,7 +135,7 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) { SNODE_DECODE_OVER: if (terrno != 0) { mError("snode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -200,7 +200,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S createReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -215,7 +215,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -227,7 +227,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -242,7 +242,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } @@ -352,7 +352,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn dropReq.dnodeId = pDnode->id; int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -367,7 +367,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); return -1; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index c167a44e9d..7799ac7562 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -98,7 +98,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } @@ -106,7 +106,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } @@ -114,7 +114,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfSmas; ++i) { SSchema *pSchema = &pStb->pSmas[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } @@ -175,9 +175,9 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, STB_DECODE_OVER) - pStb->pColumns = calloc(pStb->numOfColumns, sizeof(SSchema)); - pStb->pTags = calloc(pStb->numOfTags, sizeof(SSchema)); - pStb->pSmas = calloc(pStb->numOfSmas, sizeof(SSchema)); + pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); + pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema)); + pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema)); if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) { goto STB_DECODE_OVER; } @@ -185,7 +185,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } @@ -193,7 +193,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } @@ -201,13 +201,13 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfSmas; ++i) { SSchema *pSchema = &pStb->pSmas[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } if (pStb->commentLen > 0) { - pStb->comment = calloc(pStb->commentLen, 1); + pStb->comment = taosMemoryCalloc(pStb->commentLen, 1); if (pStb->comment == NULL) goto STB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_DECODE_OVER) } @@ -218,10 +218,10 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { STB_DECODE_OVER: if (terrno != 0) { mError("stb:%s, failed to decode from raw:%p since %s", pStb->name, pRaw, terrstr()); - tfree(pStb->pColumns); - tfree(pStb->pTags); - tfree(pStb->comment); - tfree(pRow); + taosMemoryFreeClear(pStb->pColumns); + taosMemoryFreeClear(pStb->pTags); + taosMemoryFreeClear(pStb->comment); + taosMemoryFreeClear(pRow); return NULL; } @@ -236,9 +236,9 @@ static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { mTrace("stb:%s, perform delete action, row:%p", pStb->name, pStb); - tfree(pStb->pColumns); - tfree(pStb->pTags); - tfree(pStb->comment); + taosMemoryFreeClear(pStb->pColumns); + taosMemoryFreeClear(pStb->pTags); + taosMemoryFreeClear(pStb->comment); return 0; } @@ -248,9 +248,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { taosWLockLatch(&pOld->lock); if (pOld->numOfColumns < pNew->numOfColumns) { - void *pColumns = malloc(pNew->numOfColumns * sizeof(SSchema)); + void *pColumns = taosMemoryMalloc(pNew->numOfColumns * sizeof(SSchema)); if (pColumns != NULL) { - free(pOld->pColumns); + taosMemoryFree(pOld->pColumns); pOld->pColumns = pColumns; } else { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -260,9 +260,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } if (pOld->numOfTags < pNew->numOfTags) { - void *pTags = malloc(pNew->numOfTags * sizeof(SSchema)); + void *pTags = taosMemoryMalloc(pNew->numOfTags * sizeof(SSchema)); if (pTags != NULL) { - free(pOld->pTags); + taosMemoryFree(pOld->pTags); pOld->pTags = pTags; } else { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -272,9 +272,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } if (pOld->numOfSmas < pNew->numOfSmas) { - void *pSmas = malloc(pNew->numOfSmas * sizeof(SSchema)); + void *pSmas = taosMemoryMalloc(pNew->numOfSmas * sizeof(SSchema)); if (pSmas != NULL) { - free(pOld->pSmas); + taosMemoryFree(pOld->pSmas); pOld->pSmas = pSmas; } else { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -284,9 +284,9 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { } if (pOld->commentLen < pNew->commentLen) { - void *comment = malloc(pNew->commentLen); + void *comment = taosMemoryMalloc(pNew->commentLen); if (comment != NULL) { - free(pOld->comment); + taosMemoryFree(pOld->comment); pOld->comment = comment; } else { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -353,7 +353,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.stbCfg.pTagSchema = pStb->pTags; int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = malloc(contLen); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -380,7 +380,7 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, req.suid = pStb->uid; int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); - SMsgHead *pHead = malloc(contLen); + SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -508,7 +508,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_CREATE_STB; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -547,7 +547,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -584,7 +584,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.numOfSmas = pCreate->numOfSmas; stbObj.commentLen = pCreate->commentLen; if (stbObj.commentLen > 0) { - stbObj.comment = calloc(stbObj.commentLen, 1); + stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1); if (stbObj.comment == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -592,9 +592,9 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen); } - stbObj.pColumns = malloc(stbObj.numOfColumns * sizeof(SSchema)); - stbObj.pTags = malloc(stbObj.numOfTags * sizeof(SSchema)); - stbObj.pSmas = malloc(stbObj.numOfSmas * sizeof(SSchema)); + stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema)); + stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema)); + stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema)); if (stbObj.pColumns == NULL || stbObj.pTags == NULL || stbObj.pSmas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -777,8 +777,8 @@ static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *col } static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { - pNew->pTags = calloc(pNew->numOfTags, sizeof(SSchema)); - pNew->pColumns = calloc(pNew->numOfColumns, sizeof(SSchema)); + pNew->pTags = taosMemoryCalloc(pNew->numOfTags, sizeof(SSchema)); + pNew->pColumns = taosMemoryCalloc(pNew->numOfColumns, sizeof(SSchema)); if (pNew->pTags == NULL || pNew->pColumns == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -1071,7 +1071,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.contLen = contLen; action.msgType = TDMT_VND_ALTER_STB; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -1140,8 +1140,8 @@ static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAltertbReq *p ALTER_STB_OVER: mndTransDrop(pTrans); - tfree(stbObj.pTags); - tfree(stbObj.pColumns); + taosMemoryFreeClear(stbObj.pTags); + taosMemoryFreeClear(stbObj.pColumns); return code; } @@ -1251,7 +1251,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * action.msgType = TDMT_VND_DROP_STB; action.acceptableCode = TSDB_CODE_VND_TB_NOT_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pReq); + taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -1348,7 +1348,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa taosRLockLatch(&pStb->lock); int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; - pRsp->pSchemas = calloc(totalCols, sizeof(SSchema)); + pRsp->pSchemas = taosMemoryCalloc(totalCols, sizeof(SSchema)); if (pRsp->pSchemas == NULL) { taosRUnLockLatch(&pStb->lock); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1498,7 +1498,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int return -1; } - void *pRsp = malloc(rspLen); + void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { tFreeSTableMetaBatchRsp(&batchMetaRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1613,7 +1613,7 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32 SDbObj* pDb = NULL; if (strlen(pShow->db) > 0) { pDb = mndAcquireDb(pMnode, pShow->db); - if (pDb == NULL) return 0; + if (pDb == NULL) return terrno; } while (numOfRows < rows) { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index e3d77ced0d..c02fec0a5f 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -14,7 +14,6 @@ */ #include "mndStream.h" -#include "parser.h" #include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" @@ -26,6 +25,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "parser.h" #include "tname.h" #define MND_STREAM_VER_NUMBER 1 @@ -84,7 +84,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SSdbRaw *pRaw = sdbAllocRaw(SDB_STREAM, MND_STREAM_VER_NUMBER, size); if (pRaw == NULL) goto STREAM_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto STREAM_ENCODE_OVER; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); @@ -102,7 +102,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { terrno = TSDB_CODE_SUCCESS; STREAM_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -135,7 +135,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { int32_t tlen; int32_t dataPos = 0; SDB_GET_INT32(pRaw, dataPos, &tlen, STREAM_DECODE_OVER); - buf = malloc(tlen + 1); + buf = taosMemoryMalloc(tlen + 1); if (buf == NULL) goto STREAM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER); @@ -148,10 +148,10 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; STREAM_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -248,23 +248,22 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) { int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { SNode *pAst = NULL; -#if 1 // TODO: remove debug info later - printf("ast = %s\n", ast); -#endif + if (nodesStringToNode(ast, &pAst) < 0) { return -1; } -#if 1 - SSchemaWrapper sw = {0}; - qExtractResultSchema(pAst, (int32_t*)&sw.nCols, &sw.pSchema); + if (qExtractResultSchema(pAst, (int32_t *)&pStream->outputSchema.nCols, &pStream->outputSchema.pSchema) != 0) { + return -1; + } + +#if 1 printf("|"); - for (int i = 0; i < sw.nCols; i++) { - printf(" %15s |", (char *)sw.pSchema[i].name); + for (int i = 0; i < pStream->outputSchema.nCols; i++) { + printf(" %15s |", (char *)pStream->outputSchema.pSchema[i].name); } printf("\n=======================================================\n"); - pStream->ColAlias = NULL; #endif if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, &pStream->physicalPlan)) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a4909dfe37..08a88a19ec 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -94,7 +94,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { tDeleteSMqSubscribeObj(pSub); - free(pSub); + taosMemoryFree(pSub); return NULL; } @@ -102,7 +102,7 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSMqSubscribeObj(pSub); - free(pSub); + taosMemoryFree(pSub); return NULL; } #endif @@ -118,7 +118,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume }; int32_t tlen = tEncodeSMqMVRebReq(NULL, &req); - void *buf = malloc(sizeof(SMsgHead) + tlen); + void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -157,7 +157,7 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } @@ -169,7 +169,7 @@ static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsum req.consumerId = pConsumerEp->consumerId; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *buf = malloc(sizeof(SMsgHead) + tlen); + void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -203,7 +203,7 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } @@ -229,7 +229,7 @@ static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg) { SMqOffset *pOffset = &req.offsets[i]; SMqVgOffsets *pVgOffset = taosHashGet(pHash, &pOffset->vgId, sizeof(int32_t)); if (pVgOffset == NULL) { - pVgOffset = malloc(sizeof(SMqVgOffsets)); + pVgOffset = taosMemoryMalloc(sizeof(SMqVgOffsets)); if (pVgOffset == NULL) { return -1; } @@ -407,7 +407,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { int32_t removeSz = taosArrayGetSize(pConsumer->recentRemovedTopics); for (int32_t i = 0; i < removeSz; i++) { char *topicName = taosArrayGet(pConsumer->recentRemovedTopics, i); - free(topicName); + taosMemoryFree(topicName); } taosArrayClear(pConsumer->recentRemovedTopics); } @@ -441,7 +441,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { if (pIter == NULL) break; SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); - tfree(pRebSub->key); + taosMemoryFreeClear(pRebSub->key); mInfo("mq rebalance subscription: %s", pSub->key); @@ -762,8 +762,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { } mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); - tfree(topic); - tfree(cgroup); + taosMemoryFreeClear(topic); + taosMemoryFreeClear(cgroup); } // rebalance condition2 : imbalance assignment } @@ -836,7 +836,7 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT strcpy(req.cgroup, cgroup); strcpy(req.topicName, pTopic->name); int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *buf = malloc(sizeof(SMsgHead) + tlen); + void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -860,7 +860,7 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(buf); + taosMemoryFree(buf); return -1; } return 0; @@ -877,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); if (pRaw == NULL) goto SUB_ENCODE_OVER; - buf = malloc(tlen); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto SUB_ENCODE_OVER; void *abuf = buf; @@ -892,7 +892,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_SUCCESS; SUB_ENCODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -925,7 +925,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); - buf = malloc(tlen + 1); + buf = taosMemoryMalloc(tlen + 1); if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); @@ -937,10 +937,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; SUB_DECODE_OVER: - tfree(buf); + taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -1140,7 +1140,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { } } - if (oldSub) taosArrayDestroyEx(oldSub, free); + if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void*))taosMemoryFree); // persist consumerObj SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer); diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 7993d7df9d..d6c1b6c94f 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -88,7 +88,7 @@ static int32_t mndProcessTelemTimer(SNodeMsg* pReq) { char* pCont = mndBuildTelemetryReport(pMnode); if (pCont != NULL) { taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont), HTTP_FLAT); - free(pCont); + taosMemoryFree(pCont); } taosWUnLockLatch(&pMgmt->lock); return 0; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 95dce5d8f1..fa2ba4bfc0 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -23,6 +23,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "parser.h" #include "tname.h" #define MND_TOPIC_VER_NUMBER 1 @@ -85,6 +86,16 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER); + int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema); + void *swBuf = taosMemoryMalloc(swLen); + if (swBuf == NULL) { + goto TOPIC_ENCODE_OVER; + } + void *aswBuf = swBuf; + taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema); + SDB_SET_INT32(pRaw, dataPos, swLen, TOPIC_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, swBuf, swLen, TOPIC_ENCODE_OVER); + SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER); @@ -129,11 +140,11 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); - pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char)); + pTopic->sql = taosMemoryCalloc(pTopic->sqlLen + 1, sizeof(char)); SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - pTopic->logicalPlan = calloc(len + 1, sizeof(char)); + pTopic->logicalPlan = taosMemoryCalloc(len + 1, sizeof(char)); if (pTopic->logicalPlan == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; @@ -141,14 +152,25 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); - pTopic->physicalPlan = calloc(len + 1, sizeof(char)); + pTopic->physicalPlan = taosMemoryCalloc(len + 1, sizeof(char)); if (pTopic->physicalPlan == NULL) { - free(pTopic->logicalPlan); + taosMemoryFree(pTopic->logicalPlan); terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; } SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); + void *buf = taosMemoryMalloc(len); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto TOPIC_DECODE_OVER; + } + SDB_GET_BINARY(pRaw, dataPos, buf, len, TOPIC_DECODE_OVER); + if (taosDecodeSSchemaWrapper(buf, &pTopic->schema) == NULL) { + goto TOPIC_DECODE_OVER; + } + SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER); terrno = TSDB_CODE_SUCCESS; @@ -156,7 +178,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { TOPIC_DECODE_OVER: if (terrno != TSDB_CODE_SUCCESS) { mError("topic:%s, failed to decode from raw:%p since %s", pTopic->name, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -214,7 +236,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { int32_t contLen = sizeof(SDDropTopicReq); - SDDropTopicReq *pDrop = calloc(1, contLen); + SDDropTopicReq *pDrop = taosMemoryCalloc(1, contLen); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -283,10 +305,18 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq topicObj.physicalPlan = pPlanStr; } + SNode *pAst = NULL; + if (nodesStringToNode(pCreate->ast, &pAst) < 0) { + return -1; + } + if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) { + return -1; + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); return -1; } mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); @@ -294,7 +324,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq SSdbRaw *pRedoRaw = mndTopicActionEncode(&topicObj); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); mndTransDrop(pTrans); return -1; } @@ -302,12 +332,12 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); mndTransDrop(pTrans); return -1; } - tfree(pPlanStr); + taosMemoryFreeClear(pPlanStr); mndTransDrop(pTrans); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index aa39c740ad..3314211437 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -255,7 +255,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < redoLogNum; ++i) { SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) - pData = malloc(dataLen); + pData = taosMemoryMalloc(dataLen); if (pData == NULL) goto TRANS_DECODE_OVER; mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); @@ -265,7 +265,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < undoLogNum; ++i) { SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) - pData = malloc(dataLen); + pData = taosMemoryMalloc(dataLen); if (pData == NULL) goto TRANS_DECODE_OVER; mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); @@ -275,7 +275,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < commitLogNum; ++i) { SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) - pData = malloc(dataLen); + pData = taosMemoryMalloc(dataLen); if (pData == NULL) goto TRANS_DECODE_OVER; mTrace("raw:%p, is created", pData); SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); @@ -288,7 +288,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) - action.pCont = malloc(action.contLen); + action.pCont = taosMemoryMalloc(action.contLen); if (action.pCont == NULL) goto TRANS_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto TRANS_DECODE_OVER; @@ -300,7 +300,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) - action.pCont = malloc(action.contLen); + action.pCont = taosMemoryMalloc(action.contLen); if (action.pCont == NULL) goto TRANS_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto TRANS_DECODE_OVER; @@ -315,9 +315,9 @@ TRANS_DECODE_OVER: if (terrno != 0) { mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); mndTransDropData(pTrans); - tfree(pRow); - tfree(pData); - tfree(action.pCont); + taosMemoryFreeClear(pRow); + taosMemoryFreeClear(pData); + taosMemoryFreeClear(action.pCont); return NULL; } @@ -428,7 +428,7 @@ static void mndTransDropData(STrans *pTrans) { mndTransDropActions(pTrans->redoActions); mndTransDropActions(pTrans->undoActions); if (pTrans->rpcRsp != NULL) { - free(pTrans->rpcRsp); + taosMemoryFree(pTrans->rpcRsp); pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; } @@ -472,7 +472,7 @@ static void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { } STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq) { - STrans *pTrans = calloc(1, sizeof(STrans)); + STrans *pTrans = taosMemoryCalloc(1, sizeof(STrans)); if (pTrans == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to create transaction since %s", terrstr()); @@ -517,7 +517,7 @@ static void mndTransDropActions(SArray *pArray) { int32_t size = taosArrayGetSize(pArray); for (int32_t i = 0; i < size; ++i) { STransAction *pAction = taosArrayGet(pArray, i); - tfree(pAction->pCont); + taosMemoryFreeClear(pAction->pCont); } taosArrayDestroy(pArray); @@ -527,7 +527,7 @@ void mndTransDrop(STrans *pTrans) { if (pTrans != NULL) { mndTransDropData(pTrans); mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); - tfree(pTrans); + taosMemoryFreeClear(pTrans); } } @@ -762,7 +762,7 @@ static void mndTransSendRpcRsp(STrans *pTrans) { if (rpcCont != NULL) { memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); } - free(pTrans->rpcRsp); + taosMemoryFree(pTrans->rpcRsp); mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index b7d32c01d9..ff34c26c4a 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -201,7 +201,7 @@ USER_DECODE_OVER: mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr()); taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0b31f7d06c..d36617822d 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -146,7 +146,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { VG_DECODE_OVER: if (terrno != 0) { mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); - tfree(pRow); + taosMemoryFreeClear(pRow); return NULL; } @@ -250,7 +250,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return NULL; } - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -274,7 +274,7 @@ void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgOb return NULL; } - void *pReq = malloc(contLen); + void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -376,7 +376,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { SArray *pArray = NULL; SVgObj *pVgroups = NULL; - pVgroups = calloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); + pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto ALLOC_VGROUP_OVER; @@ -430,7 +430,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); ALLOC_VGROUP_OVER: - if (code != 0) free(pVgroups); + if (code != 0) taosMemoryFree(pVgroups); taosArrayDestroy(pArray); return code; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index c4d389a379..67b7d6dd45 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -39,6 +39,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndQuery.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 @@ -79,7 +80,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { .pCont = pReq, .contLen = contLen, }; - tmsgPutToQueue(&pMnode->msgCb, QUERY_QUEUE, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); @@ -91,7 +92,7 @@ static void mndPullupTelem(void *param, void *tmrId) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; - tmsgPutToQueue(&pMnode->msgCb, QUERY_QUEUE, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); @@ -217,6 +218,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { // if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1; if (mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem) != 0) return -1; if (mndAllocStep(pMnode, "mnode-timer", NULL, mndCleanupTimer) != 0) return -1; @@ -283,7 +285,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { mDebug("start to open mnode in %s", path); - SMnode *pMnode = calloc(1, sizeof(SMnode)); + SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode)); if (pMnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to open mnode since %s", terrstr()); @@ -295,7 +297,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep)); if (pMnode->pSteps == NULL) { - free(pMnode); + taosMemoryFree(pMnode); terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to open mnode since %s", terrstr()); return NULL; @@ -346,8 +348,8 @@ void mndClose(SMnode *pMnode) { if (pMnode != NULL) { mDebug("start to close mnode"); mndCleanupSteps(pMnode, -1); - tfree(pMnode->path); - tfree(pMnode); + taosMemoryFreeClear(pMnode->path); + taosMemoryFreeClear(pMnode); mDebug("mnode is closed"); } } diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 9ddb3594bb..b37ec85387 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -29,7 +29,7 @@ class MndTestTrans : public ::testing::Test { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; - void* buffer = malloc(size); + void* buffer = taosMemoryMalloc(size); int32_t readLen = taosReadFile(pFile, buffer, size); if (readLen < 0 || readLen == size) { ASSERT(1); @@ -43,7 +43,7 @@ class MndTestTrans : public ::testing::Test { if (writeLen < 0 || writeLen == readLen) { ASSERT(1); } - free(buffer); + taosMemoryFree(buffer); taosFsyncFile(pFile); taosCloseFile(&pFile); taosMsleep(1000); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 26809ea059..c645dae5b5 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -21,7 +21,7 @@ static int32_t sdbCreateDir(SSdb *pSdb); SSdb *sdbInit(SSdbOpt *pOption) { mDebug("start to init sdb in %s", pOption->path); - SSdb *pSdb = calloc(1, sizeof(SSdb)); + SSdb *pSdb = taosMemoryCalloc(1, sizeof(SSdb)); if (pSdb == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to init sdb since %s", terrstr()); @@ -67,15 +67,15 @@ void sdbCleanup(SSdb *pSdb) { sdbWriteFile(pSdb); if (pSdb->currDir != NULL) { - tfree(pSdb->currDir); + taosMemoryFreeClear(pSdb->currDir); } if (pSdb->syncDir != NULL) { - tfree(pSdb->syncDir); + taosMemoryFreeClear(pSdb->syncDir); } if (pSdb->tmpDir != NULL) { - tfree(pSdb->tmpDir); + taosMemoryFreeClear(pSdb->tmpDir); } for (ESdbType i = 0; i < SDB_MAX; ++i) { @@ -102,7 +102,7 @@ void sdbCleanup(SSdb *pSdb) { mDebug("sdb table:%d is cleaned up", i); } - free(pSdb); + taosMemoryFree(pSdb); mDebug("sdb is cleaned up"); } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 2849da5c2e..c11025beed 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -137,7 +137,7 @@ int32_t sdbReadFile(SSdb *pSdb) { int32_t readLen = 0; int64_t ret = 0; - SSdbRaw *pRaw = malloc(SDB_MAX_SIZE); + SSdbRaw *pRaw = taosMemoryMalloc(SDB_MAX_SIZE); if (pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed read file since %s", terrstr()); @@ -150,7 +150,7 @@ int32_t sdbReadFile(SSdb *pSdb) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - free(pRaw); + taosMemoryFree(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, terrstr()); return 0; @@ -159,7 +159,7 @@ int32_t sdbReadFile(SSdb *pSdb) { if (sdbReadFileHead(pSdb, pFile) != 0) { mError("failed to read file:%s head since %s", file, terrstr()); pSdb->curVer = -1; - free(pRaw); + taosMemoryFree(pRaw); taosCloseFile(&pFile); return -1; } diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index e37559808e..c3aaf562be 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -17,7 +17,7 @@ #include "sdbInt.h" SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { - SSdbRaw *pRaw = calloc(1, dataLen + sizeof(SSdbRaw)); + SSdbRaw *pRaw = taosMemoryCalloc(1, dataLen + sizeof(SSdbRaw)); if (pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -33,7 +33,7 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { void sdbFreeRaw(SSdbRaw *pRaw) { mTrace("raw:%p, is freed", pRaw); - free(pRaw); + taosMemoryFree(pRaw); } int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { diff --git a/source/dnode/mnode/sdb/src/sdbRow.c b/source/dnode/mnode/sdb/src/sdbRow.c index e51dd48dcd..ac86a72155 100644 --- a/source/dnode/mnode/sdb/src/sdbRow.c +++ b/source/dnode/mnode/sdb/src/sdbRow.c @@ -17,7 +17,7 @@ #include "sdbInt.h" SSdbRow *sdbAllocRow(int32_t objSize) { - SSdbRow *pRow = calloc(1, objSize + sizeof(SSdbRow)); + SSdbRow *pRow = taosMemoryCalloc(1, objSize + sizeof(SSdbRow)); if (pRow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -46,5 +46,5 @@ void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) { sdbPrintOper(pSdb, pRow, "freeRow"); mTrace("row:%p, is freed", pRow->pObj); - tfree(pRow); + taosMemoryFreeClear(pRow); } diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index bc1a602b52..0d1e890097 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -19,14 +19,14 @@ #include "qworker.h" SQnode *qndOpen(const SQnodeOpt *pOption) { - SQnode *pQnode = calloc(1, sizeof(SQnode)); + SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); if (NULL == pQnode) { qError("calloc SQnode failed"); return NULL; } if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { - tfree(pQnode); + taosMemoryFreeClear(pQnode); return NULL; } @@ -37,7 +37,7 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { void qndClose(SQnode *pQnode) { qWorkerDestroy((void **)&pQnode->pQuery); - free(pQnode); + taosMemoryFree(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index 0c63e35e87..f177bda47a 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -13,4 +13,5 @@ target_link_libraries( PRIVATE common PRIVATE util PRIVATE qcom + PRIVATE stream ) diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 519b94cf46..2802537dcd 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -22,6 +22,7 @@ #include "tmsg.h" #include "tqueue.h" #include "trpc.h" +#include "tstream.h" #include "snode.h" diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index afa5930821..a5937b625f 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -18,14 +18,14 @@ #include "tuuid.h" SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { - SSnode *pSnode = calloc(1, sizeof(SSnode)); + SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); if (pSnode == NULL) { return NULL; } pSnode->msgCb = pOption->msgCb; pSnode->pMeta = sndMetaNew(); if (pSnode->pMeta == NULL) { - free(pSnode); + taosMemoryFree(pSnode); return NULL; } return pSnode; @@ -33,19 +33,19 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { void sndClose(SSnode *pSnode) { sndMetaDelete(pSnode->pMeta); - free(pSnode); + taosMemoryFree(pSnode); } int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } SStreamMeta *sndMetaNew() { - SStreamMeta *pMeta = calloc(1, sizeof(SStreamMeta)); + SStreamMeta *pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { return NULL; } pMeta->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (pMeta->pHash == NULL) { - free(pMeta); + taosMemoryFree(pMeta); return NULL; } return pMeta; @@ -53,12 +53,12 @@ SStreamMeta *sndMetaNew() { void sndMetaDelete(SStreamMeta *pMeta) { taosHashCleanup(pMeta->pHash); - free(pMeta); + taosMemoryFree(pMeta); } int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) { - for (int i = 0; i < pTask->numOfRunners; i++) { - pTask->runner[i].executor = qCreateStreamExecTaskInfo(pTask->qmsg, NULL); + for (int i = 0; i < pTask->exec.numOfRunners; i++) { + pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); } return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *)); } @@ -72,19 +72,19 @@ int32_t sndMetaRemoveTask(SStreamMeta *pMeta, int32_t taskId) { if (pTask == NULL) { return -1; } - free(pTask->qmsg); + taosMemoryFree(pTask->exec.qmsg); // TODO:free executor - free(pTask); + taosMemoryFree(pTask); return taosHashRemove(pMeta->pHash, &taskId, sizeof(int32_t)); } static int32_t sndProcessTaskExecReq(SSnode *pSnode, SRpcMsg *pMsg) { - SStreamExecMsgHead *pHead = pMsg->pCont; - int32_t taskId = pHead->streamTaskId; - SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId); - if (pTask == NULL) { - return -1; - } + /*SStreamExecMsgHead *pHead = pMsg->pCont;*/ + /*int32_t taskId = pHead->streamTaskId;*/ + /*SStreamTask *pTask = sndMetaGetTask(pSnode->pMeta, taskId);*/ + /*if (pTask == NULL) {*/ + /*return -1;*/ + /*}*/ return 0; } @@ -94,7 +94,7 @@ void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { // operator exec if (pMsg->msgType == TDMT_SND_TASK_DEPLOY) { void *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - SStreamTask *pTask = malloc(sizeof(SStreamTask)); + SStreamTask *pTask = taosMemoryMalloc(sizeof(SStreamTask)); if (pTask == NULL) { ASSERT(0); return; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index f026f8331b..bdc8a71b04 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -43,6 +43,7 @@ target_link_libraries( PUBLIC wal PUBLIC scheduler PUBLIC executor + PUBLIC stream PUBLIC qworker PUBLIC sync ) diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 735db64263..f1083c0d91 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -276,6 +276,8 @@ int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGro */ void tsdbCleanupReadHandle(tsdbReaderT queryHandle); +int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tsdbMemory.h b/source/dnode/vnode/src/inc/tsdbMemory.h index df4df0053f..69976fc078 100644 --- a/source/dnode/vnode/src/inc/tsdbMemory.h +++ b/source/dnode/vnode/src/inc/tsdbMemory.h @@ -30,7 +30,7 @@ static void taosTMemset(void *ptr, int c); static FORCE_INLINE void *taosTMalloc(size_t size) { if (size <= 0) return NULL; - void *ret = malloc(size + sizeof(size_t)); + void *ret = taosMemoryMalloc(size + sizeof(size_t)); if (ret == NULL) return NULL; *(size_t *)ret = size; @@ -58,7 +58,7 @@ static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) { void * tptr = (void *)((char *)ptr - sizeof(size_t)); size_t tsize = size + sizeof(size_t); - void* tptr1 = realloc(tptr, tsize); + void* tptr1 = taosMemoryRealloc(tptr, tsize); if (tptr1 == NULL) return NULL; tptr = tptr1; @@ -69,7 +69,7 @@ static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) { static FORCE_INLINE void* taosTZfree(void* ptr) { if (ptr) { - free((void*)((char*)ptr - sizeof(size_t))); + taosMemoryFree((void*)((char*)ptr - sizeof(size_t))); } return NULL; } diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index e4bad9e94b..b91c6cd9e3 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -233,7 +233,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { // save sma info int32_t len = tEncodeTSma(NULL, pSmaCfg); - pBuf = calloc(len, 1); + pBuf = taosMemoryCalloc(1, len); if (pBuf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -254,7 +254,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { metaDBULock(pMeta->pDB); // release - tfree(pBuf); + taosMemoryFreeClear(pBuf); return 0; } @@ -285,7 +285,7 @@ static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI32(buf, pSchema->colId); + tlen += taosEncodeFixedI16(buf, pSchema->colId); tlen += taosEncodeFixedI32(buf, pSchema->bytes); tlen += taosEncodeString(buf, pSchema->name); } @@ -297,11 +297,11 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { SSchema *pSchema; buf = taosDecodeFixedU32(buf, &pSW->nCols); - pSW->pSchema = (SSchema *)malloc(sizeof(SSchema) * pSW->nCols); + pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosDecodeFixedI32(buf, &pSchema->colId); + buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeFixedI32(buf, &pSchema->bytes); buf = taosDecodeStringTo(buf, pSchema->name); } @@ -311,7 +311,7 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { static SMetaDB *metaNewDB() { SMetaDB *pDB = NULL; - pDB = (SMetaDB *)calloc(1, sizeof(*pDB)); + pDB = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDB)); if (pDB == NULL) { return NULL; } @@ -328,7 +328,7 @@ static void metaFreeDB(SMetaDB *pDB) { #if IMPL_WITH_LOCK taosThreadRwlockDestroy(&pDB->rwlock); #endif - free(pDB); + taosMemoryFree(pDB); } } @@ -463,7 +463,7 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey DBT *pDbt; if (pTbCfg->type == META_CHILD_TABLE) { - // pDbt = calloc(2, sizeof(DBT)); + // pDbt = taosMemoryCalloc(2, sizeof(DBT)); // // First key is suid // pDbt[0].data = &(pTbCfg->ctbCfg.suid); @@ -516,6 +516,7 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid); tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag); } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO } else { ASSERT(0); } @@ -538,6 +539,7 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag)); } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO } else { ASSERT(0); } @@ -545,11 +547,11 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { } static void metaClearTbCfg(STbCfg *pTbCfg) { - tfree(pTbCfg->name); + taosMemoryFreeClear(pTbCfg->name); if (pTbCfg->type == META_SUPER_TABLE) { tdFreeSchema(pTbCfg->stbCfg.pTagSchema); } else if (pTbCfg->type == META_CHILD_TABLE) { - tfree(pTbCfg->ctbCfg.pTag); + taosMemoryFreeClear(pTbCfg->ctbCfg.pTag); } } @@ -574,7 +576,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { } // Decode - pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + pTbCfg = (STbCfg *)taosMemoryMalloc(sizeof(*pTbCfg)); if (pTbCfg == NULL) { return NULL; } @@ -606,7 +608,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { // Decode *uid = *(tb_uid_t *)(pkey.data); - pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + pTbCfg = (STbCfg *)taosMemoryMalloc(sizeof(*pTbCfg)); if (pTbCfg == NULL) { return NULL; } @@ -636,13 +638,13 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { } // Decode - pCfg = (STSma *)calloc(1, sizeof(STSma)); + pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma)); if (pCfg == NULL) { return NULL; } if (tDecodeTSma(value.data, pCfg) == NULL) { - tfree(pCfg); + taosMemoryFreeClear(pCfg); return NULL; } @@ -675,7 +677,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo // Decode the schema pBuf = value.data; - pSW = malloc(sizeof(*pSW)); + pSW = taosMemoryMalloc(sizeof(*pSW)); metaDecodeSchema(pBuf, pSW); return pSW; @@ -689,7 +691,7 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { SMTbCursor *pTbCur = NULL; SMetaDB *pDB = pMeta->pDB; - pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur)); + pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); if (pTbCur == NULL) { return NULL; } @@ -722,7 +724,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { if (pTbCur->pCur) { pTbCur->pCur->close(pTbCur->pCur); } - free(pTbCur); + taosMemoryFree(pTbCur); } } @@ -737,8 +739,8 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { pBuf = value.data; metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { - free(tbCfg.name); - free(tbCfg.stbCfg.pTagSchema); + taosMemoryFree(tbCfg.name); + taosMemoryFree(tbCfg.stbCfg.pTagSchema); continue; } else if (tbCfg.type == META_CHILD_TABLE) { kvRowFree(tbCfg.ctbCfg.pTag); @@ -792,7 +794,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMetaDB *pDB = pMeta->pDB; int ret; - pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur)); + pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); if (pCtbCur == NULL) { return NULL; } @@ -800,7 +802,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { pCtbCur->suid = uid; ret = pDB->pCtbIdx->cursor(pDB->pCtbIdx, NULL, &(pCtbCur->pCur), 0); if (ret != 0) { - free(pCtbCur); + taosMemoryFree(pCtbCur); return NULL; } @@ -813,7 +815,7 @@ void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { pCtbCur->pCur->close(pCtbCur->pCur); } - free(pCtbCur); + taosMemoryFree(pCtbCur); } } @@ -849,7 +851,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { SMetaDB *pDB = pMeta->pDB; int ret; - pCur = (SMSmaCursor *)calloc(1, sizeof(*pCur)); + pCur = (SMSmaCursor *)taosMemoryCalloc(1, sizeof(*pCur)); if (pCur == NULL) { return NULL; } @@ -858,7 +860,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { // TODO: lock? ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0); if (ret != 0) { - free(pCur); + taosMemoryFree(pCur); return NULL; } @@ -871,7 +873,7 @@ void metaCloseSmaCurosr(SMSmaCursor *pCur) { pCur->pCur->close(pCur->pCur); } - free(pCur); + taosMemoryFree(pCur); } } @@ -896,14 +898,14 @@ const char *metaSmaCursorNext(SMSmaCursor *pCur) { STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { STSmaWrapper *pSW = NULL; - pSW = calloc(1, sizeof(*pSW)); + pSW = taosMemoryCalloc(1, sizeof(*pSW)); if (pSW == NULL) { return NULL; } SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); if (pCur == NULL) { - free(pSW); + taosMemoryFree(pSW); return NULL; } @@ -915,11 +917,11 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { // TODO: lock? if (pCur->pCur->pget(pCur->pCur, &skey, NULL, &pval, DB_NEXT) == 0) { ++pSW->number; - STSma *tptr = (STSma *)realloc(pSW->tSma, pSW->number * sizeof(STSma)); + STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { metaCloseSmaCurosr(pCur); tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); return NULL; } pSW->tSma = tptr; @@ -927,7 +929,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) { metaCloseSmaCurosr(pCur); tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); return NULL; } continue; diff --git a/source/dnode/vnode/src/meta/metaMain.c b/source/dnode/vnode/src/meta/metaMain.c index ad87b2de9e..690b96bbb0 100644 --- a/source/dnode/vnode/src/meta/metaMain.c +++ b/source/dnode/vnode/src/meta/metaMain.c @@ -69,7 +69,7 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorF SMeta *pMeta; size_t psize = strlen(path); - pMeta = (SMeta *)calloc(1, sizeof(*pMeta)); + pMeta = (SMeta *)taosMemoryCalloc(1, sizeof(*pMeta)); if (pMeta == NULL) { return NULL; } @@ -88,8 +88,8 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorF static void metaFree(SMeta *pMeta) { if (pMeta) { - tfree(pMeta->path); - free(pMeta); + taosMemoryFreeClear(pMeta->path); + taosMemoryFree(pMeta); } } diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 4a65cf277b..f4b450b4a8 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -47,7 +47,7 @@ int metaOpenDB(SMeta *pMeta) { TDB * pCtbIdx; int ret; - pDb = (SMetaDB *)calloc(1, sizeof(*pDb)); + pDb = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 43554f923e..91cbc2cff8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -14,10 +14,10 @@ */ #include "tcompare.h" +#include "tdatablock.h" #include "tqInt.h" #include "tqMetaStore.h" - -void tqDebugShowSSData(SArray* dataBlocks); +#include "tstream.h" int32_t tqInit() { return tqPushMgrInit(); } @@ -25,7 +25,7 @@ void tqCleanUp() { tqPushMgrCleanUp(); } STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { - STQ* pTq = malloc(sizeof(STQ)); + STQ* pTq = taosMemoryMalloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; @@ -43,9 +43,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STq } #endif pTq->tqMeta = - tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, free, 0); + tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, (FTqDelete)taosMemoryFree, 0); if (pTq->tqMeta == NULL) { - free(pTq); + taosMemoryFree(pTq); #if 0 allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); #endif @@ -56,7 +56,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STq pTq->tqPushMgr = tqPushMgrOpen(); if (pTq->tqPushMgr == NULL) { // free store - free(pTq); + taosMemoryFree(pTq); return NULL; } #endif @@ -68,15 +68,15 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STq void tqClose(STQ* pTq) { if (pTq) { - tfree(pTq->path); - free(pTq); + taosMemoryFreeClear(pTq->path); + taosMemoryFree(pTq); } // TODO } int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) { if (msgType != TDMT_VND_SUBMIT) return 0; - void* data = malloc(msgLen); + void* data = taosMemoryMalloc(msgLen); if (data == NULL) { return -1; } @@ -95,7 +95,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi if (pusher->type == TQ_PUSHER_TYPE__STREAM) { STqStreamPusher* streamPusher = (STqStreamPusher*)pusher; // repack - STqStreamToken* token = malloc(sizeof(STqStreamToken)); + STqStreamToken* token = taosMemoryMalloc(sizeof(STqStreamToken)); if (token == NULL) { taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -199,9 +199,9 @@ int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead int32_t sz = tEncodeSTqConsumer(NULL, pConsumer); if (sz > (*ppHead)->ssize) { - void* tmpPtr = realloc(*ppHead, sizeof(STqSerializedHead) + sz); + void* tmpPtr = taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sz); if (tmpPtr == NULL) { - free(*ppHead); + taosMemoryFree(*ppHead); terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } @@ -218,7 +218,7 @@ int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsumer** ppConsumer) { const void* str = pHead->content; - *ppConsumer = calloc(1, sizeof(STqConsumer)); + *ppConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); if (*ppConsumer == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -392,7 +392,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { tDecodeSMqSetCVgReq(msg, &req); /*printf("vg %d set to consumer from %ld to %ld\n", req.vgId, req.oldConsumerId, req.newConsumerId);*/ - STqConsumer* pConsumer = calloc(1, sizeof(STqConsumer)); + STqConsumer* pConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); if (pConsumer == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -403,10 +403,10 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { pConsumer->consumerId = req.consumerId; pConsumer->epoch = 0; - STqTopic* pTopic = calloc(1, sizeof(STqTopic)); + STqTopic* pTopic = taosMemoryCalloc(1, sizeof(STqTopic)); if (pTopic == NULL) { taosArrayDestroy(pConsumer->topics); - free(pConsumer); + taosMemoryFree(pConsumer); return -1; } strcpy(pTopic->topicName, req.topicName); @@ -441,22 +441,28 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { - ASSERT(parallel <= 8); - pTask->numOfRunners = parallel; + if (pTask->execType == TASK_EXEC__NONE) return 0; + + pTask->exec.numOfRunners = parallel; + pTask->exec.runners = taosMemoryCalloc(parallel, sizeof(SStreamRunner)); + if (pTask->exec.runners == NULL) { + return -1; + } for (int32_t i = 0; i < parallel; i++) { STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); SReadHandle handle = { .reader = pReadHandle, .meta = pTq->pVnodeMeta, }; - pTask->runner[i].inputHandle = pReadHandle; - pTask->runner[i].executor = qCreateStreamExecTaskInfo(pTask->qmsg, &handle); + pTask->exec.runners[i].inputHandle = pReadHandle; + pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); + ASSERT(pTask->exec.runners[i].executor); } return 0; } int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { - SStreamTask* pTask = malloc(sizeof(SStreamTask)); + SStreamTask* pTask = taosMemoryMalloc(sizeof(SStreamTask)); if (pTask == NULL) { return -1; } @@ -467,93 +473,15 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { } tCoderClear(&decoder); - tqExpandTask(pTq, pTask, 8); + if (tqExpandTask(pTq, pTask, 4) < 0) { + ASSERT(0); + } + taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask)); return 0; } -static char* formatTimestamp(char* buf, int64_t val, int precision) { - time_t tt; - int32_t ms = 0; - if (precision == TSDB_TIME_PRECISION_NANO) { - tt = (time_t)(val / 1000000000); - ms = val % 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - tt = (time_t)(val / 1000000); - ms = val % 1000000; - } else { - tt = (time_t)(val / 1000); - ms = val % 1000; - } - - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif - if (tt <= 0 && ms < 0) { - tt--; - if (precision == TSDB_TIME_PRECISION_NANO) { - ms += 1000000000; - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - ms += 1000000; - } else { - ms += 1000; - } - } - - struct tm* ptm = localtime(&tt); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); - - if (precision == TSDB_TIME_PRECISION_NANO) { - sprintf(buf + pos, ".%09d", ms); - } else if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", ms); - } else { - sprintf(buf + pos, ".%03d", ms); - } - - return buf; -} -void tqDebugShowSSData(SArray* dataBlocks) { - char pBuf[128]; - int32_t sz = taosArrayGetSize(dataBlocks); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; - int32_t rows = pDataBlock->info.rows; - for (int32_t j = 0; j < rows; j++) { - printf("|"); - for (int32_t k = 0; k < colNum; k++) { - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); - printf(" %25s |", pBuf); - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - printf(" %15d |", *(int32_t*)var); - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - printf(" %15ld |", *(int64_t*)var); - break; - } - } - printf("\n"); - } - } -} - int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen) { void* pIter = NULL; @@ -561,50 +489,9 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen) { pIter = taosHashIterate(pTq->pStreamTasks, pIter); if (pIter == NULL) break; SStreamTask* pTask = (SStreamTask*)pIter; - if (!pTask->sourceType) continue; - int32_t workerId = 0; - void* exec = pTask->runner[workerId].executor; - qSetStreamInput(exec, data, STREAM_DATA_TYPE_SUBMIT_BLOCK); - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - while (1) { - SSDataBlock* output; - uint64_t ts; - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(false); - } - if (output == NULL) { - break; - } - taosArrayPush(pRes, output); - } - if (pTask->sinkType) { - // write back - /*printf("reach end\n");*/ - tqDebugShowSSData(pRes); - } else { - int32_t tlen = sizeof(SStreamExecMsgHead) + tEncodeDataBlocks(NULL, pRes); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - void* abuf = POINTER_SHIFT(buf, sizeof(SStreamExecMsgHead)); - tEncodeDataBlocks(abuf, pRes); - tmsg_t type; - - if (pTask->nextOpDst == STREAM_NEXT_OP_DST__VND) { - type = TDMT_VND_TASK_EXEC; - } else { - type = TDMT_SND_TASK_EXEC; - } - - SRpcMsg reqMsg = { - .pCont = buf, - .contLen = tlen, - .code = 0, - .msgType = type, - }; - tmsgSendReq(&pTq->pVnode->msgCb, &pTask->NextOpEp, &reqMsg); + if (streamExecTask(pTask, &pTq->pVnode->msgCb, data, STREAM_DATA_TYPE_SUBMIT_BLOCK, 0) < 0) { + // TODO } } return 0; @@ -612,33 +499,12 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen) { int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg) { SStreamTaskExecReq* pReq = msg->pCont; + int32_t taskId = pReq->taskId; + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + ASSERT(pTask); - int32_t taskId = pReq->head.streamTaskId; - int32_t workerType = pReq->head.workerType; - - SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - // assume worker id is 1 - int32_t workerId = 1; - void* exec = pTask->runner[workerId].executor; - int32_t sz = taosArrayGetSize(pReq->data); - printf("input data:\n"); - tqDebugShowSSData(pReq->data); - SArray* pRes = taosArrayInit(0, sizeof(void*)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* input = taosArrayGet(pReq->data, i); - SSDataBlock* output; - uint64_t ts; - qSetStreamInput(exec, input, STREAM_DATA_TYPE_SSDATA_BLOCK); - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(0); - } - if (output == NULL) { - break; - } - taosArrayPush(pRes, &output); + if (streamExecTask(pTask, &pTq->pVnode->msgCb, pReq->data, STREAM_DATA_TYPE_SSDATA_BLOCK, 0) < 0) { + // TODO } - printf("output data:\n"); - tqDebugShowSSData(pRes); - return 0; } diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index ce00c98ff9..505687755d 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -70,7 +70,7 @@ static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter, int32_t tqConfigFlag) { - STqMetaStore* pMeta = calloc(1, sizeof(STqMetaStore)); + STqMetaStore* pMeta = taosMemoryCalloc(1, sizeof(STqMetaStore)); if (pMeta == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; @@ -79,10 +79,10 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F // concat data file name and index file name size_t pathLen = strlen(path); - pMeta->dirPath = malloc(pathLen + 1); + pMeta->dirPath = taosMemoryMalloc(pathLen + 1); if (pMeta->dirPath == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - free(pMeta); + taosMemoryFree(pMeta); return NULL; } strcpy(pMeta->dirPath, path); @@ -104,7 +104,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } pMeta->pIdxFile = pIdxFile; - pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); + pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; @@ -129,7 +129,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F // read idx file and load into memory STqIdxPageBuf idxBuf; - STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); + STqSerializedHead* serializedObj = taosMemoryMalloc(TQ_PAGE_SIZE); if (serializedObj == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; } @@ -145,7 +145,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F ASSERT(idxBuf.head.writeOffset == idxRead); // loop read every entry for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - STqMetaList* pNode = calloc(1, sizeof(STqMetaList)); + STqMetaList* pNode = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; // TODO: free memory @@ -154,7 +154,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); if (allocated < pNode->handle.serializedSize) { - void* ptr = realloc(serializedObj, pNode->handle.serializedSize); + void* ptr = taosMemoryRealloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; // TODO: free memory @@ -225,11 +225,11 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pBucketNode->handle.valueInTxn); } - free(pBucketNode); + taosMemoryFree(pBucketNode); } } } - free(serializedObj); + taosMemoryFree(serializedObj); return pMeta; } @@ -252,13 +252,13 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; - free(pNode); + taosMemoryFree(pNode); pNode = next; } } - free(pMeta->dirPath); - free(pMeta->unpersistHead); - free(pMeta); + taosMemoryFree(pMeta->dirPath); + taosMemoryFree(pMeta->unpersistHead); + taosMemoryFree(pMeta); return 0; } @@ -277,14 +277,14 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; - free(pNode); + taosMemoryFree(pNode); pNode = next; } } - free(pMeta->unpersistHead); + taosMemoryFree(pMeta->unpersistHead); taosRemoveDir(pMeta->dirPath); - free(pMeta->dirPath); - free(pMeta); + taosMemoryFree(pMeta->dirPath); + taosMemoryFree(pMeta); return 0; } @@ -293,7 +293,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { int64_t* bufPtr = (int64_t*)idxBuf.buffer; STqMetaList* pHead = pMeta->unpersistHead; STqMetaList* pNode = pHead->unpersistNext; - STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead)); + STqSerializedHead* pSHead = taosMemoryMalloc(sizeof(STqSerializedHead)); if (pSHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -383,12 +383,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { ASSERT(pBucketNode->next == pNode); pBucketNode->next = pNode->next; } - free(pNode); + taosMemoryFree(pNode); } } // write left bytes - free(pSHead); + taosMemoryFree(pSHead); // TODO: write new version in tfile if ((char*)bufPtr != idxBuf.buffer) { int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); @@ -416,7 +416,7 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu pNode = pNode->next; } } - STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); + STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -488,7 +488,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va pNode = pNode->next; } } - STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); + STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; @@ -504,7 +504,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); } int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { - void* vmem = malloc(vsize); + void* vmem = taosMemoryMalloc(vsize); if (vmem == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 4115cb7313..20270950cd 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -31,7 +31,7 @@ struct STqOffsetStore { }; STqOffsetStore* STqOffsetOpen(STqOffsetCfg* pCfg) { - STqOffsetStore* pStore = malloc(sizeof(STqOffsetStore)); + STqOffsetStore* pStore = taosMemoryMalloc(sizeof(STqOffsetStore)); if (pStore == NULL) { return NULL; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 4186f29e2a..7851c071c3 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -32,7 +32,7 @@ void tqPushMgrCleanUp() { } STqPushMgr* tqPushMgrOpen() { - STqPushMgr* mgr = malloc(sizeof(STqPushMgr)); + STqPushMgr* mgr = taosMemoryMalloc(sizeof(STqPushMgr)); if (mgr == NULL) { return NULL; } @@ -42,11 +42,11 @@ STqPushMgr* tqPushMgrOpen() { void tqPushMgrClose(STqPushMgr* pushMgr) { taosHashCleanup(pushMgr->pHash); - free(pushMgr); + taosMemoryFree(pushMgr); } STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) { - STqClientPusher* clientPusher = malloc(sizeof(STqClientPusher)); + STqClientPusher* clientPusher = taosMemoryMalloc(sizeof(STqClientPusher)); if (clientPusher == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -57,7 +57,7 @@ STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t c clientPusher->ttl = ttl; if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(clientPusher); + taosMemoryFree(clientPusher); // TODO send rsp back return NULL; } @@ -65,7 +65,7 @@ STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t c } STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) { - STqStreamPusher* streamPusher = malloc(sizeof(STqStreamPusher)); + STqStreamPusher* streamPusher = taosMemoryMalloc(sizeof(STqStreamPusher)); if (streamPusher == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -77,7 +77,7 @@ STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - free(streamPusher); + taosMemoryFree(streamPusher); return NULL; } return streamPusher; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 690787984c..8c161e4c8b 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -17,7 +17,7 @@ #include "vnode.h" STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { - STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); + STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle)); if (pReadHandle == NULL) { return NULL; } @@ -130,8 +130,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int32_t colNeed = 0; while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - int16_t colIdSchema = pColSchema->colId; - int16_t colIdNeed = *(int16_t*)taosArrayGet(pHandle->pColIdList, colNeed); + col_id_t colIdSchema = pColSchema->colId; + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pHandle->pColIdList, colNeed); if (colIdSchema < colIdNeed) { colMeta++; } else if (colIdSchema > colIdNeed) { @@ -143,7 +143,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { colInfo.info.colId = pColSchema->colId; colInfo.info.type = pColSchema->type; - colInfo.pData = calloc(1, sz); + colInfo.pData = taosMemoryCalloc(1, sz); if (colInfo.pData == NULL) { // TODO free taosArrayDestroy(pArray); @@ -159,7 +159,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int j = 0; for (int32_t i = 0; i < colNumNeed; i++) { - int16_t colId = *(int16_t*)taosArrayGet(pHandle->pColIdList, i); + col_id_t colId = *(col_id_t*)taosArrayGet(pHandle->pColIdList, i); while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { j++; } @@ -173,7 +173,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { colInfo.info.colId = colId; colInfo.info.type = pColSchema->type; - colInfo.pData = calloc(1, sz); + colInfo.pData = taosMemoryCalloc(1, sz); if (colInfo.pData == NULL) { // TODO free taosArrayDestroy(pArray); diff --git a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c index ee279abf47..c8f4cd642a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbBDBImpl.c @@ -51,7 +51,7 @@ void tsdbCloseDBF(SDBFile *pDBF) { tsdbCloseBDBDb(pDBF->pDB); pDBF->pDB = NULL; } - tfree(pDBF->path); + taosMemoryFreeClear(pDBF->path); } int32_t tsdbOpenBDBEnv(DB_ENV **ppEnv, const char *path) { @@ -159,7 +159,7 @@ void *tsdbGetSmaDataByKey(SDBFile *pDBF, void* key, uint32_t keySize, uint32_t * return NULL; } - result = calloc(1, value1.size); + result = taosMemoryCalloc(1, value1.size); if (result == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 37403f1a11..eb8df61051 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -403,7 +403,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { STbData *pTbData; pCommith->niters = SL_SIZE(pMem->pSlIdx); - pCommith->iters = (SCommitIter *)calloc(pCommith->niters, sizeof(SCommitIter)); + pCommith->iters = (SCommitIter *)taosMemoryCalloc(pCommith->niters, sizeof(SCommitIter)); if (pCommith->iters == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; @@ -424,7 +424,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pCommitIter->pIter = tSkipListCreateIter(pTbData->pData); tSkipListIterNext(pCommitIter->pIter); - pCommitIter->pTable = (STable *)malloc(sizeof(STable)); + pCommitIter->pTable = (STable *)taosMemoryMalloc(sizeof(STable)); pCommitIter->pTable->uid = pTbData->uid; pCommitIter->pTable->tid = pTbData->uid; pCommitIter->pTable->pSchema = metaGetTbTSchema(pRepo->pMeta, pTbData->uid, 0); @@ -439,10 +439,10 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) { for (int i = 1; i < pCommith->niters; i++) { tSkipListDestroyIter(pCommith->iters[i].pIter); tdFreeSchema(pCommith->iters[i].pTable->pSchema); - free(pCommith->iters[i].pTable); + taosMemoryFree(pCommith->iters[i].pTable); } - free(pCommith->iters); + taosMemoryFree(pCommith->iters); pCommith->iters = NULL; pCommith->niters = 0; } @@ -985,7 +985,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // SKVRecord *pRecord; // void *pBuf = NULL; -// pBuf = malloc((size_t)maxBufSize); +// pBuf = taosMemoryMalloc((size_t)maxBufSize); // if (pBuf == NULL) { // goto _err; // } @@ -1006,7 +1006,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // } // if (pRecord->size > maxBufSize) { // maxBufSize = pRecord->size; -// void* tmp = realloc(pBuf, (size_t)maxBufSize); +// void* tmp = taosMemoryRealloc(pBuf, (size_t)maxBufSize); // if (tmp == NULL) { // goto _err; // } @@ -1059,7 +1059,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // pfs->metaCacheComp = NULL; // } -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // ASSERT(mf.info.nDels == 0); // ASSERT(mf.info.tombSize == 0); @@ -1369,7 +1369,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF } } - // Update pBlock membership vairables + // Update pBlock membership variables pBlock->last = isLast; pBlock->offset = offset; pBlock->algorithm = pCfg->compression; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index fa867543b0..aa235f88de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -134,7 +134,7 @@ static void *tsdbDecodeFSStatus(STsdb*pRepo, void *buf, SFSStatus *pStatus) { } static SFSStatus *tsdbNewFSStatus(int maxFSet) { - SFSStatus *pStatus = (SFSStatus *)calloc(1, sizeof(*pStatus)); + SFSStatus *pStatus = (SFSStatus *)taosMemoryCalloc(1, sizeof(*pStatus)); if (pStatus == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -145,7 +145,7 @@ static SFSStatus *tsdbNewFSStatus(int maxFSet) { pStatus->df = taosArrayInit(maxFSet, sizeof(SDFileSet)); if (pStatus->df == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - free(pStatus); + taosMemoryFree(pStatus); return NULL; } @@ -155,7 +155,7 @@ static SFSStatus *tsdbNewFSStatus(int maxFSet) { static SFSStatus *tsdbFreeFSStatus(SFSStatus *pStatus) { if (pStatus) { pStatus->df = taosArrayDestroy(pStatus->df); - free(pStatus); + taosMemoryFree(pStatus); } return NULL; @@ -197,7 +197,7 @@ STsdbFS *tsdbNewFS(const STsdbCfg *pCfg) { int maxFSet = TSDB_MAX_FSETS(keep, days); STsdbFS *pfs; - pfs = (STsdbFS *)calloc(1, sizeof(*pfs)); + pfs = (STsdbFS *)taosMemoryCalloc(1, sizeof(*pfs)); if (pfs == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -206,7 +206,7 @@ STsdbFS *tsdbNewFS(const STsdbCfg *pCfg) { int code = taosThreadRwlockInit(&(pfs->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); - free(pfs); + taosMemoryFree(pfs); return NULL; } @@ -242,7 +242,7 @@ void *tsdbFreeFS(STsdbFS *pfs) { pfs->metaCache = NULL; pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus); taosThreadRwlockDestroy(&(pfs->lock)); - free(pfs); + taosMemoryFree(pfs); } return NULL; @@ -853,7 +853,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // } // if (recoverMeta) { -// pBuf = malloc((size_t)maxBufSize); +// pBuf = taosMemoryMalloc((size_t)maxBufSize); // if (pBuf == NULL) { // terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; // tsdbCloseMFile(pMFile); @@ -865,7 +865,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { // tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), // tstrerror(terrno)); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -874,7 +874,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // if (nread < 0) { // tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), // tstrerror(terrno)); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -883,7 +883,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", // REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); // terrno = TSDB_CODE_TDB_FILE_CORRUPTED; -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -891,7 +891,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) { // tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid, // tstrerror(terrno)); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // tsdbCloseMFile(pMFile); // return -1; // } @@ -903,7 +903,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { // } // tsdbCloseMFile(pMFile); -// tfree(pBuf); +// taosMemoryFreeClear(pBuf); // return 0; // } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 00e97c7b61..271616e9c2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -85,7 +85,7 @@ void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) { strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN); TSDB_FILE_SET_CLOSED(pMFile); - tfree(aname); + taosMemoryFreeClear(aname); return buf; } @@ -119,10 +119,10 @@ int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) { // Try to create directory recursively char *s = strdup(TFILE_REL_NAME(&(pMFile->f))); if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) { - tfree(s); + taosMemoryFreeClear(s); return -1; } - tfree(s); + taosMemoryFreeClear(s); pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); if (pMFile->fd < 0) { @@ -352,7 +352,7 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { buf = taosDecodeString(buf, &aname); strncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN); TSDB_FILE_SET_CLOSED(pDFile); - tfree(aname); + taosMemoryFreeClear(aname); return buf; } @@ -366,10 +366,10 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); if (tfsMkdirRecurAt(pRepo->pTfs, taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) { - tfree(s); + taosMemoryFreeClear(s); return -1; } - tfree(s); + taosMemoryFreeClear(s); pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pDFile->pFile == NULL) { @@ -692,7 +692,7 @@ int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype } } - tfree(p); + taosMemoryFreeClear(p); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 131e92cab6..526109f796 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -68,7 +68,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMeta *pMeta, STfs *pTfs) { STsdb *pTsdb = NULL; - pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); + pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -93,8 +93,8 @@ static void tsdbFree(STsdb *pTsdb) { tsdbFreeSmaEnv(pTsdb->pRSmaEnv); tsdbFreeSmaEnv(pTsdb->pTSmaEnv); tsdbFreeFS(pTsdb->fs); - tfree(pTsdb->path); - free(pTsdb); + taosMemoryFreeClear(pTsdb->path); + taosMemoryFree(pTsdb); } } @@ -507,7 +507,7 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t magic = pFile->info.magic; char *tfname = strdup(fname); sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); + taosMemoryFreeClear(tfname); } else { if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { SFile *pFile = &pFGroup->files[0]; @@ -516,17 +516,17 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t magic = pFile->info.magic; char *tfname = strdup(fname); sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); + taosMemoryFreeClear(tfname); } else { return 0; } } } } else { // get the named file at the specified index. If not there, return 0 - fname = malloc(256); + fname = taosMemoryMalloc(256); sprintf(fname, "%s/vnode/vnode%d/%s", tfsGetPrimaryPath(pRepo->pTfs), REPO_ID(pRepo), name); if (access(fname, F_OK) != 0) { - tfree(fname); + taosMemoryFreeClear(fname); return 0; } if (*index == TSDB_META_FILE_INDEX) { // get meta file @@ -536,19 +536,19 @@ uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); tsdbGetFileInfoImpl(tfname, &magic, size); } - tfree(fname); + taosMemoryFreeClear(fname); return magic; } if (stat(fname, &fState) < 0) { - tfree(fname); + taosMemoryFreeClear(fname); return 0; } *size = fState.st_size; // magic = *size; - tfree(fname); + taosMemoryFreeClear(fname); return magic; #endif } @@ -674,7 +674,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { - STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo)); + STsdbRepo *pRepo = (STsdbRepo *)taosMemoryCalloc(1, sizeof(*pRepo)); if (pRepo == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -748,7 +748,7 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { // tsdbFreeMemTable(pRepo->imem); tsem_destroy(&(pRepo->readyToCommit)); taosThreadMutexDestroy(&pRepo->mutex); - free(pRepo); + taosMemoryFree(pRepo); } } @@ -820,7 +820,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea goto out; } - pBlockStatis = calloc(numColumns, sizeof(SDataStatis)); + pBlockStatis = taosMemoryCalloc(numColumns, sizeof(SDataStatis)); if (pBlockStatis == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; err = -1; @@ -886,7 +886,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea // save not-null column uint16_t bytes = IS_VAR_DATA_TYPE(pCol->type) ? varDataTLen(pColData) : pCol->bytes; SDataCol *pLastCol = &(pTable->lastCols[idx]); - pLastCol->pData = malloc(bytes); + pLastCol->pData = taosMemoryMalloc(bytes); pLastCol->bytes = bytes; pLastCol->colId = pCol->colId; memcpy(pLastCol->pData, value, bytes); @@ -907,7 +907,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea out: taosTZfree(row); - tfree(pBlockStatis); + taosMemoryFreeClear(pBlockStatis); if (err == 0 && numColumns <= pTable->restoreColumnNum) { pTable->hasRestoreLastColumn = true; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index a6df63dfa0..34dba5d3ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -25,7 +25,7 @@ static char * tsdbTbDataGetUid(const void *arg); static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row); STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { - STsdbMemTable *pMemTable = (STsdbMemTable *)calloc(1, sizeof(*pMemTable)); + STsdbMemTable *pMemTable = (STsdbMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable)); if (pMemTable == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -38,7 +38,7 @@ STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { pMemTable->nRow = 0; pMemTable->pMA = pTsdb->pmaf->create(pTsdb->pmaf); if (pMemTable->pMA == NULL) { - free(pMemTable); + taosMemoryFree(pMemTable); return NULL; } @@ -47,7 +47,7 @@ STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { tSkipListCreate(5, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t), tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid); if (pMemTable->pSlIdx == NULL) { pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); - free(pMemTable); + taosMemoryFree(pMemTable); return NULL; } @@ -55,7 +55,7 @@ STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) { if (pMemTable->pHashIdx == NULL) { pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); tSkipListDestroy(pMemTable->pSlIdx); - free(pMemTable); + taosMemoryFree(pMemTable); return NULL; } @@ -69,7 +69,7 @@ void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) { if (pMemTable->pMA) { pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA); } - free(pMemTable); + taosMemoryFree(pMemTable); } } @@ -227,6 +227,34 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey return 0; } +int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) { + ASSERT(pMsg != NULL); + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SSubmitBlkIter blkIter = {0}; + STSRow *row = NULL; + + terrno = TSDB_CODE_SUCCESS; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; + while (true) { + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + if (pBlock == NULL) break; + + pBlock->uid = htobe64(pBlock->uid); + pBlock->suid = htobe64(pBlock->suid); + pBlock->sversion = htonl(pBlock->sversion); + pBlock->dataLen = htonl(pBlock->dataLen); + pBlock->schemaLen = htonl(pBlock->schemaLen); + pBlock->numOfRows = htons(pBlock->numOfRows); + } + + if (terrno != TSDB_CODE_SUCCESS) return -1; + return 0; +} + static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; @@ -376,7 +404,7 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p } static STbData *tsdbNewTbData(tb_uid_t uid) { - STbData *pTbData = (STbData *)calloc(1, sizeof(*pTbData)); + STbData *pTbData = (STbData *)taosMemoryCalloc(1, sizeof(*pTbData)); if (pTbData == NULL) { return NULL; } @@ -397,14 +425,14 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { // tkeyComparFn, skipListCreateFlags, tsdbGetTsTupleKey); // if (pTableData->pData == NULL) { // terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - // free(pTableData); + // taosMemoryFree(pTableData); // return NULL; // } pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTbData->pData == NULL) { - free(pTbData); + taosMemoryFree(pTbData); return NULL; } @@ -414,7 +442,7 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { static void tsdbFreeTbData(STbData *pTbData) { if (pTbData) { tSkipListDestroy(pTbData->pData); - free(pTbData); + taosMemoryFree(pTbData); } } @@ -582,7 +610,7 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot, SArray *pATab pSnapshot->mem = &(pSnapshot->mtable); - pSnapshot->mem->tData = (STableData **)calloc(pSnapshot->omem->maxTables, sizeof(STableData *)); + pSnapshot->mem->tData = (STableData **)taosMemoryCalloc(pSnapshot->omem->maxTables, sizeof(STableData *)); if (pSnapshot->mem->tData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; taosRUnLockLatch(&(pSnapshot->omem->latch)); @@ -629,7 +657,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot) { tsdbFreeTableData(pTableData); } } - tfree(pSnapshot->mem->tData); + taosMemoryFreeClear(pSnapshot->mem->tData); tsdbUnRefMemTable(pRepo, pSnapshot->omem); } @@ -990,10 +1018,10 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, STSRow* ro TSDB_WLOCK_TABLE(pTable); SDataCol *pDataCol = &(pLatestCols[idx]); if (pDataCol->pData == NULL) { - pDataCol->pData = malloc(pTCol->bytes); + pDataCol->pData = taosMemoryMalloc(pTCol->bytes); pDataCol->bytes = pTCol->bytes; } else if (pDataCol->bytes < pTCol->bytes) { - pDataCol->pData = realloc(pDataCol->pData, pTCol->bytes); + pDataCol->pData = taosMemoryRealloc(pDataCol->pData, pTCol->bytes); pDataCol->bytes = pTCol->bytes; } // the actual value size diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index af3d454a86..bac5255d17 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -355,7 +355,7 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* } static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, uint64_t taskId) { - STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle)); + STsdbReadHandle* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReadHandle)); if (pReadHandle == NULL) { goto _end; } @@ -388,7 +388,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, if (pCond->numOfCols > 0) { // allocate buffer in order to load data blocks from file - pReadHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); + pReadHandle->statis = taosMemoryCalloc(pCond->numOfCols, sizeof(SDataStatis)); if (pReadHandle->statis == NULL) { goto _end; } @@ -597,7 +597,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr assert(pGroupList); size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); - STableGroupInfo* pNew = calloc(1, sizeof(STableGroupInfo)); + STableGroupInfo* pNew = taosMemoryCalloc(1, sizeof(STableGroupInfo)); pNew->pGroupList = taosArrayInit(numOfGroup, POINTER_BYTES); for(int32_t i = 0; i < numOfGroup; ++i) { @@ -1009,7 +1009,7 @@ static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, i if (pCheckInfo->compSize < (int32_t)compIndex->len) { assert(compIndex->len > 0); - char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); + char* t = taosMemoryRealloc(pCheckInfo->pCompInfo, compIndex->len); if (t == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; code = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1974,15 +1974,15 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { } static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { - tfree(pSupporter->numOfBlocksPerTable); - tfree(pSupporter->blockIndexArray); + taosMemoryFreeClear(pSupporter->numOfBlocksPerTable); + taosMemoryFreeClear(pSupporter->blockIndexArray); for (int32_t i = 0; i < numOfTables; ++i) { STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i]; - tfree(pBlockInfo); + taosMemoryFreeClear(pBlockInfo); } - tfree(pSupporter->pDataBlockInfo); + taosMemoryFreeClear(pSupporter->pDataBlockInfo); } static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { @@ -2021,7 +2021,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu if (pTsdbReadHandle->allocSize < size) { pTsdbReadHandle->allocSize = (int32_t)size; - char* tmp = realloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); + char* tmp = taosMemoryRealloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); if (tmp == NULL) { return TSDB_CODE_TDB_OUT_OF_MEMORY; } @@ -2037,9 +2037,9 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; - sup.numOfBlocksPerTable = calloc(1, sizeof(int32_t) * numOfTables); - sup.blockIndexArray = calloc(1, sizeof(int32_t) * numOfTables); - sup.pDataBlockInfo = calloc(1, POINTER_BYTES * numOfTables); + sup.numOfBlocksPerTable = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + sup.blockIndexArray = taosMemoryCalloc(1, sizeof(int32_t) * numOfTables); + sup.pDataBlockInfo = taosMemoryCalloc(1, POINTER_BYTES * numOfTables); if (sup.numOfBlocksPerTable == NULL || sup.blockIndexArray == NULL || sup.pDataBlockInfo == NULL) { cleanBlockOrderSupporter(&sup, 0); @@ -2058,7 +2058,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu SBlock* pBlock = pTableCheck->pCompInfo->blocks; sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks; - char* buf = malloc(sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); + char* buf = taosMemoryMalloc(sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); if (buf == NULL) { cleanBlockOrderSupporter(&sup, numOfQualTables); return TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -2128,7 +2128,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu tsdbDebug("%p %d data blocks sort completed, %s", pTsdbReadHandle, cnt, pTsdbReadHandle->idStr); cleanBlockOrderSupporter(&sup, numOfTables); - free(pTree); + taosMemoryFree(pTree); return TSDB_CODE_SUCCESS; } @@ -2536,12 +2536,12 @@ static void destroyHelper(void* param) { // tQueryInfo* pInfo = (tQueryInfo*)param; // if (pInfo->optr != TSDB_RELATION_IN) { -// tfree(pInfo->q); +// taosMemoryFreeClear(pInfo->q); // } else { // taosHashCleanup((SHashObj *)(pInfo->q)); // } - free(param); + taosMemoryFree(param); } #define TSDB_PREV_ROW 0x1 @@ -2616,7 +2616,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // return false; // } mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->tableId, NULL, NULL, true); - tfree(pRow); + taosMemoryFreeClear(pRow); // update the last key value pCheckInfo->lastKey = key + step; @@ -2904,7 +2904,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // // SColumnInfoData colInfo = {{0}, 0}; // colInfo.info = pCol->info; -// colInfo.pData = calloc(1, pCol->info.bytes); +// colInfo.pData = taosMemoryCalloc(1, pCol->info.bytes); // if (colInfo.pData == NULL) { // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // goto out_of_memory; @@ -2923,7 +2923,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MAX}; // } // -// cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); +// cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo)); // if (cond.colList == NULL) { // terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // goto out_of_memory; @@ -2935,7 +2935,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // } // // pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->idStr, pMemRef); -// tfree(cond.colList); +// taosMemoryFreeClear(cond.colList); // // // current table, only one table // STableCheckInfo* pCurrent = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); @@ -3294,7 +3294,7 @@ void filterPrepare(void* expr, void* param) { return; } - pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); + pExpr->_node.info = taosMemoryCalloc(1, sizeof(tQueryInfo)); STSchema* pTSSchema = (STSchema*) param; tQueryInfo* pInfo = pExpr->_node.info; @@ -3327,7 +3327,7 @@ void filterPrepare(void* expr, void* param) { size = pSchema->bytes; } // to make sure tonchar does not cause invalid write, since the '\0' needs at least sizeof(TdUcs4) space. - pInfo->q = calloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + pInfo->q = taosMemoryCalloc(1, size + TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); tVariantDump(pCond, pInfo->q, pSchema->type, true); } } @@ -3613,7 +3613,7 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch // if (tagExpr != NULL) { // CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, tagExpr, NULL); // tExprNode* tbnameExpr = expr; -// expr = calloc(1, sizeof(tExprNode)); +// expr = taosMemoryCalloc(1, sizeof(tExprNode)); // if (expr == NULL) { // THROW( TSDB_CODE_TDB_OUT_OF_MEMORY ); // } @@ -3727,7 +3727,7 @@ static void* doFreeColumnInfoData(SArray* pColumnInfoData) { size_t cols = taosArrayGetSize(pColumnInfoData); for (int32_t i = 0; i < cols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pColumnInfoData, i); - tfree(pColInfo->pData); + taosMemoryFreeClear(pColInfo->pData); } taosArrayDestroy(pColumnInfoData); @@ -3740,7 +3740,7 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i); destroyTableMemIterator(p); - tfree(p->pCompInfo); + taosMemoryFreeClear(p->pCompInfo); } taosArrayDestroy(pTableCheckInfo); @@ -3757,8 +3757,8 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); taosArrayDestroy(pTsdbReadHandle->defaultLoadColumn); - tfree(pTsdbReadHandle->pDataBlockInfo); - tfree(pTsdbReadHandle->statis); + taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo); + taosMemoryFreeClear(pTsdbReadHandle->statis); if (!emptyQueryTimewindow(pTsdbReadHandle)) { // tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); @@ -3783,7 +3783,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %s", pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->idStr); - tfree(pTsdbReadHandle); + taosMemoryFreeClear(pTsdbReadHandle); } #if 0 @@ -3842,15 +3842,15 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { - pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start = taosMemoryCalloc(1, sizeof(SEndPoint)); pCond->start->optr = queryColInfo->optr; pCond->start->v = queryColInfo->q; } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - pCond->end = calloc(1, sizeof(SEndPoint)); + pCond->end = taosMemoryCalloc(1, sizeof(SEndPoint)); pCond->end->optr = queryColInfo->optr; pCond->end->v = queryColInfo->q; } else if (optr == TSDB_RELATION_IN) { - pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start = taosMemoryCalloc(1, sizeof(SEndPoint)); pCond->start->optr = queryColInfo->optr; pCond->start->v = queryColInfo->q; } else if (optr == TSDB_RELATION_LIKE) { @@ -3996,8 +3996,8 @@ static void queryIndexedColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SAr } } - free(cond.start); - free(cond.end); + taosMemoryFree(cond.start); + taosMemoryFree(cond.end); tSkipListDestroyIter(iter); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index a10252e286..07eafd6df0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -38,9 +38,10 @@ typedef enum { } ESmaStorageLevel; typedef struct { - STsdb *pTsdb; - SDBFile dFile; - int32_t interval; // interval with the precision of DB + STsdb *pTsdb; + SDBFile dFile; + SSDataBlock *pData; // sma data + int32_t interval; // interval with the precision of DB } STSmaWriteH; typedef struct { @@ -81,6 +82,7 @@ struct SSmaStat { // expired window static int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg); +static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey); static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem); static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); @@ -97,7 +99,8 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ int32_t nMaxResult); // insert data -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, SSDataBlock *pData, int64_t interval, + int8_t intervalUnit); static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); @@ -169,7 +172,7 @@ static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) { SSmaEnv *pEnv = NULL; - pEnv = (SSmaEnv *)calloc(1, sizeof(SSmaEnv)); + pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv)); if (pEnv == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -178,7 +181,7 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) int code = taosThreadRwlockInit(&(pEnv->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); - free(pEnv); + taosMemoryFree(pEnv); return NULL; } @@ -230,8 +233,8 @@ static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaE void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { if (pSmaEnv) { tsdbDestroySmaState(pSmaEnv->pStat); - tfree(pSmaEnv->pStat); - tfree(pSmaEnv->path); + taosMemoryFreeClear(pSmaEnv->pStat); + taosMemoryFreeClear(pSmaEnv->path); taosThreadRwlockDestroy(&(pSmaEnv->lock)); tsdbCloseBDBEnv(pSmaEnv->dbEnv); } @@ -239,7 +242,7 @@ void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) { tsdbDestroySmaEnv(pSmaEnv); - tfree(pSmaEnv); + taosMemoryFreeClear(pSmaEnv); return NULL; } @@ -271,7 +274,7 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { * tsdbInitSmaStat invoked in other multithread environment later. */ if (*pSmaStat == NULL) { - *pSmaStat = (SSmaStat *)calloc(1, sizeof(SSmaStat)); + *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat)); if (*pSmaStat == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; @@ -281,7 +284,7 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if ((*pSmaStat)->smaStatItems == NULL) { - tfree(*pSmaStat); + taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; } } @@ -291,13 +294,13 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { SSmaStatItem *pItem = NULL; - pItem = (SSmaStatItem *)calloc(1, sizeof(SSmaStatItem)); + pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); if (pItem) { pItem->state = state; pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), true, HASH_ENTRY_LOCK); if (!pItem->expiredWindows) { - tfree(pItem); + taosMemoryFreeClear(pItem); } } return pItem; @@ -306,9 +309,9 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { if (pSmaStatItem != NULL) { tdDestroyTSma(pSmaStatItem->pSma); - tfree(pSmaStatItem->pSma); + taosMemoryFreeClear(pSmaStatItem->pSma); taosHashCleanup(pSmaStatItem->expiredWindows); - tfree(pSmaStatItem); + taosMemoryFreeClear(pSmaStatItem); } return NULL; } @@ -384,17 +387,12 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { return TSDB_CODE_SUCCESS; }; -static STimeWindow getActiveTimeWindowX(int64_t ts, SInterval* pInterval) { - STimeWindow tw = {0}; - tw.skey = 100; - tw.ekey = 1000; - return tw; -} static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey) { SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); if (pItem == NULL) { - pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_EXPIRED); // TODO use the real state + // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later + pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state if (pItem == NULL) { // Response to stream computing: OOM // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. @@ -406,7 +404,7 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t if (pSma == NULL) { terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; taosHashCleanup(pItem->expiredWindows); - free(pItem); + taosMemoryFree(pItem); tsdbWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", REPO_ID(pTsdb), indexUid, tstrerror(terrno)); return TSDB_CODE_FAILED; @@ -416,9 +414,12 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { // If error occurs during put smaStatItem, free the resources of pItem taosHashCleanup(pItem->expiredWindows); - free(pItem); + taosMemoryFree(pItem); return TSDB_CODE_FAILED; } + } else if ((pItem = *(SSmaStatItem **)pItem) == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; } int8_t state = TSDB_SMA_STAT_EXPIRED; @@ -430,7 +431,7 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired // windows failed to put into hash table. taosHashCleanup(pItem->expiredWindows); - tfree(pItem->pSma); + taosMemoryFreeClear(pItem->pSma); taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); return TSDB_CODE_FAILED; } @@ -491,41 +492,39 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg) { TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL); + // basic procedure + // TODO: optimization + tsdbRefSmaStat(pTsdb, pStat); SSubmitMsgIter msgIter = {0}; SSubmitBlk *pBlock = NULL; SInterval interval = {0}; - if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } - // basic procedure - // TODO: optimization - tsdbRefSmaStat(pTsdb, pStat); - while (true) { tGetSubmitMsgNext(&msgIter, &pBlock); if (pBlock == NULL) break; - int64_t suid = htobe64(pBlock->uid); STSmaWrapper *pSW = NULL; STSma *pTSma = NULL; + SSubmitBlkIter blkIter = {0}; + if (tInitSubmitBlkIter(pBlock, &blkIter) != TSDB_CODE_SUCCESS) { + tdFreeTSmaWrapper(pSW); + break; + } + while (true) { - SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIter(pBlock, &blkIter) != TSDB_CODE_SUCCESS) { - tdFreeTSmaWrapper(pSW); - break; - } STSRow *row = tGetSubmitBlkNext(&blkIter); if (row == NULL) { tdFreeTSmaWrapper(pSW); break; } if(pSW == NULL) { - if((pSW =metaGetSmaInfoByTable(REPO_META(pTsdb), suid)) == NULL) { + if((pSW =metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid)) == NULL) { break; } if((pSW->number) <= 0 || (pSW->tSma == NULL)) { @@ -542,8 +541,9 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg) { interval.sliding = pTSma->sliding; interval.slidingUnit = pTSma->slidingUnit; - STimeWindow tw = getActiveTimeWindowX(TD_ROW_KEY(row), &interval); - tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, TD_ROW_KEY(row)); + TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); + + tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey); } } @@ -802,9 +802,10 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p return TSDB_CODE_SUCCESS; } -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData) { +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, SSDataBlock *pData, int64_t interval, int8_t intervalUnit) { pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(pData->interval, pData->intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->pData = pData; return TSDB_CODE_SUCCESS; } @@ -859,10 +860,10 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe * @return int32_t */ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); - int64_t indexUid = SMA_TEST_INDEX_UID; + STsdbCfg *pCfg = REPO_CFG(pTsdb); + SSDataBlock *pData = (SSDataBlock *)msg; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; @@ -870,15 +871,15 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return terrno; } - if (pData->dataLen <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; + if (pData == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert tSma data failed since pData is NULL", REPO_ID(pTsdb)); + return terrno; } - STSmaWriteH tSmaH = {0}; - - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + if (taosArrayGetSize(pData->pDataBlock) <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + tsdbWarn("vgId:%d insert tSma data failed since pDataBlock is empty", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } @@ -897,6 +898,14 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return TSDB_CODE_FAILED; } + STSma *pSma = pItem->pSma; + + STSmaWriteH tSmaH = {0}; + + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData, pSma->interval, pSma->intervalUnit) != 0) { + return TSDB_CODE_FAILED; + } + char rPath[TSDB_FILENAME_LEN] = {0}; char aPath[TSDB_FILENAME_LEN] = {0}; snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); @@ -909,8 +918,11 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { } // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); + + +#if 0 int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file @@ -935,7 +947,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 3: reset the SSmaStat tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); - +#endif tsdbDestroyTSmaWriteH(&tSmaH); tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; @@ -1001,29 +1013,58 @@ static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, } static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); + STsdbCfg *pCfg = REPO_CFG(pTsdb); + SSDataBlock *pData = (SSDataBlock *)msg; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); return terrno; } - if (pData->dataLen <= 0) { - TASSERT(0); + if (pEnv == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + return terrno; + } + + if (pData == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert rSma data failed since pData is NULL", REPO_ID(pTsdb)); + return terrno; + } + + if (taosArrayGetSize(pData->pDataBlock) <= 0) { terrno = TSDB_CODE_INVALID_PARA; + tsdbWarn("vgId:%d insert rSma data failed since pDataBlock is empty", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } + SSmaStat *pStat = SMA_ENV_STAT(pTsdb->pTSmaEnv); + SSmaStatItem *pItem = NULL; + + tsdbRefSmaStat(pTsdb, pStat); + + if (pStat && pStat->smaStatItems) { + pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + } + + if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL) || tsdbSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + + STSma *pSma = pItem->pSma; + STSmaWriteH tSmaH = {0}; - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData, pSma->interval, pSma->intervalUnit) != 0) { return TSDB_CODE_FAILED; } - int64_t indexUid = SMA_TEST_INDEX_UID; char rPath[TSDB_FILENAME_LEN] = {0}; char aPath[TSDB_FILENAME_LEN] = {0}; snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); @@ -1035,8 +1076,9 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { } // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); + #if 0 int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file @@ -1059,8 +1101,10 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 3: reset the SSmaStat tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); +#endif tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; } @@ -1237,7 +1281,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tsdbWarn("vgId:%d get sma data v[%d]=%" PRIi64, REPO_ID(pTsdb), v, *(int64_t *)POINTER_SHIFT(result, v)); } #endif - tfree(result); // TODO: fill the result to output + taosMemoryFreeClear(result); // TODO: fill the result to output #if 0 int32_t nResult = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index 14b9a5124f..ab0da1451d 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -19,7 +19,7 @@ static SVArenaNode *vArenaNodeNew(uint64_t capacity); static void vArenaNodeFree(SVArenaNode *pNode); SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { - SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); + SVMemAllocator *pVMA = (SVMemAllocator *)taosMemoryMalloc(sizeof(*pVMA)); if (pVMA == NULL) { return NULL; } @@ -31,7 +31,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { pVMA->pNode = vArenaNodeNew(capacity); if (pVMA->pNode == NULL) { - free(pVMA); + taosMemoryFree(pVMA); return NULL; } @@ -48,7 +48,7 @@ void vmaDestroy(SVMemAllocator *pVMA) { vArenaNodeFree(pNode); } - free(pVMA); + taosMemoryFree(pVMA); } } @@ -99,7 +99,7 @@ bool vmaIsFull(SVMemAllocator *pVMA) { static SVArenaNode *vArenaNodeNew(uint64_t capacity) { SVArenaNode *pNode = NULL; - pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity); + pNode = (SVArenaNode *)taosMemoryMalloc(sizeof(*pNode) + capacity); if (pNode == NULL) { return NULL; } @@ -112,6 +112,6 @@ static SVArenaNode *vArenaNodeNew(uint64_t capacity) { static void vArenaNodeFree(SVArenaNode *pNode) { if (pNode) { - free(pNode); + taosMemoryFree(pNode); } } diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 3b43fd82f6..5be88cdc2e 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -34,7 +34,7 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocato int vnodeOpenBufPool(SVnode *pVnode) { uint64_t capacity; - if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { + if ((pVnode->pBufPool = (SVBufPool *)taosMemoryCalloc(1, sizeof(SVBufPool))) == NULL) { /* TODO */ return -1; } @@ -57,7 +57,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA); } - pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)malloc(sizeof(SMemAllocatorFactory)); + pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)taosMemoryMalloc(sizeof(SMemAllocatorFactory)); if (pVnode->pBufPool->pMAF == NULL) { // TODO: handle error return -1; @@ -71,7 +71,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { void vnodeCloseBufPool(SVnode *pVnode) { if (pVnode->pBufPool) { - tfree(pVnode->pBufPool->pMAF); + taosMemoryFreeClear(pVnode->pBufPool->pMAF); vmaDestroy(pVnode->pBufPool->inuse); while (true) { @@ -88,7 +88,7 @@ void vnodeCloseBufPool(SVnode *pVnode) { vmaDestroy(pVMA); } - free(pVnode->pBufPool); + taosMemoryFree(pVnode->pBufPool); pVnode->pBufPool = NULL; } } @@ -161,7 +161,7 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) { SVnode * pVnode = (SVnode *)(pMAF->impl); SVMAWrapper * pWrapper; - pMA = (SMemAllocator *)calloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); + pMA = (SMemAllocator *)taosMemoryCalloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); if (pMA == NULL) { return NULL; } @@ -182,7 +182,7 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) { SVnode * pVnode = pWrapper->pVnode; SVMemAllocator *pVMA = pWrapper->pVMA; - free(pMA); + taosMemoryFree(pMA); if (--pVMA->_ref.val == 0) { TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA); vmaReset(pVMA); diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index bc7a8460b8..696c5f39f6 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -24,7 +24,7 @@ int vnodeAsyncCommit(SVnode *pVnode) { vnodeWaitCommit(pVnode); vnodeBufPoolSwitch(pVnode); - SVnodeTask *pTask = (SVnodeTask *)malloc(sizeof(*pTask)); + SVnodeTask *pTask = (SVnodeTask *)taosMemoryMalloc(sizeof(*pTask)); pTask->execute = vnodeCommit; // TODO pTask->arg = pVnode; // TODO diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index 70f4117976..91c6e4d263 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -72,7 +72,7 @@ void vnodeDestroy(const char *path) { taosRemoveDir(path); } static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; - pVnode = (SVnode *)calloc(1, sizeof(*pVnode)); + pVnode = (SVnode *)taosMemoryCalloc(1, sizeof(*pVnode)); if (pVnode == NULL) { // TODO return NULL; @@ -92,8 +92,8 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { static void vnodeFree(SVnode *pVnode) { if (pVnode) { tsem_destroy(&(pVnode->canCommit)); - tfree(pVnode->path); - free(pVnode); + taosMemoryFreeClear(pVnode->path); + taosMemoryFree(pVnode); } } diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 920b5b0947..8f7d5713ab 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -29,7 +29,7 @@ int vnodeInit() { // Start commit handers vnodeMgr.nthreads = tsNumOfCommitThreads; - vnodeMgr.threads = calloc(vnodeMgr.nthreads, sizeof(TdThread)); + vnodeMgr.threads = taosMemoryCalloc(vnodeMgr.nthreads, sizeof(TdThread)); if (vnodeMgr.threads == NULL) { return -1; } @@ -65,7 +65,7 @@ void vnodeCleanup() { taosThreadJoin(vnodeMgr.threads[i], NULL); } - tfree(vnodeMgr.threads); + taosMemoryFreeClear(vnodeMgr.threads); taosThreadCondDestroy(&(vnodeMgr.hasTask)); taosThreadMutexDestroy(&(vnodeMgr.mutex)); } @@ -107,7 +107,7 @@ static void* loop(void* arg) { taosThreadMutexUnlock(&(vnodeMgr.mutex)); (*(pTask->execute))(pTask->arg); - free(pTask); + taosMemoryFree(pTask); } return NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index a3258044c8..1db17f37cb 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -31,9 +31,8 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config}; switch (pMsg->msgType) { - case TDMT_VND_QUERY: { + case TDMT_VND_QUERY: return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg); - } case TDMT_VND_QUERY_CONTINUE: return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg); default: @@ -141,7 +140,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { pTagSchema = NULL; } - metaRsp.pSchemas = calloc(nCols + nTagCols, sizeof(SSchema)); + metaRsp.pSchemas = taosMemoryCalloc(nCols + nTagCols, sizeof(SSchema)); if (metaRsp.pSchemas == NULL) { code = TSDB_CODE_VND_OUT_OF_MEMORY; goto _exit; @@ -182,19 +181,19 @@ _exit: tFreeSTableMetaRsp(&metaRsp); if (pSW != NULL) { - tfree(pSW->pSchema); - tfree(pSW); + taosMemoryFreeClear(pSW->pSchema); + taosMemoryFreeClear(pSW); } if (pTbCfg) { - tfree(pTbCfg->name); + taosMemoryFreeClear(pTbCfg->name); if (pTbCfg->type == META_SUPER_TABLE) { - free(pTbCfg->stbCfg.pTagSchema); + taosMemoryFree(pTbCfg->stbCfg.pTagSchema); } else if (pTbCfg->type == META_SUPER_TABLE) { kvRowFree(pTbCfg->ctbCfg.pTag); } - tfree(pTbCfg); + taosMemoryFreeClear(pTbCfg); } rpcMsg.handle = pMsg->handle; @@ -205,12 +204,12 @@ _exit: rpcSendResponse(&rpcMsg); - return code; + return TSDB_CODE_SUCCESS; } static void freeItemHelper(void *pItem) { char *p = *(char **)pItem; - free(p); + taosMemoryFree(p); } /** @@ -230,7 +229,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { taosArrayPush(pArray, &name); totalLen += strlen(name); } else { - tfree(name); + taosMemoryFreeClear(name); } numOfTables++; @@ -260,7 +259,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { STR_TO_VARSTR(p, n); p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); - // free(n); + // taosMemoryFree(n); } pFetchRsp->numOfRows = htonl(numOfTables); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 3ef45ecb09..02d4524bce 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -72,9 +72,12 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // TODO: maybe need to clear the request struct - free(vCreateTbReq.stbCfg.pSchema); - free(vCreateTbReq.stbCfg.pTagSchema); - free(vCreateTbReq.name); + taosMemoryFree(vCreateTbReq.stbCfg.pSchema); + taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); + taosMemoryFree(vCreateTbReq.stbCfg.pBSmaCols); + taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); + taosMemoryFree(vCreateTbReq.dbFName); + taosMemoryFree(vCreateTbReq.name); break; } case TDMT_VND_CREATE_TABLE: { @@ -101,14 +104,19 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name); } - free(pCreateTbReq->name); + taosMemoryFree(pCreateTbReq->name); + taosMemoryFree(pCreateTbReq->dbFName); if (pCreateTbReq->type == TD_SUPER_TABLE) { - free(pCreateTbReq->stbCfg.pSchema); - free(pCreateTbReq->stbCfg.pTagSchema); + taosMemoryFree(pCreateTbReq->stbCfg.pSchema); + taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema); + taosMemoryFree(pCreateTbReq->stbCfg.pBSmaCols); + taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam); } else if (pCreateTbReq->type == TD_CHILD_TABLE) { - free(pCreateTbReq->ctbCfg.pTag); + taosMemoryFree(pCreateTbReq->ctbCfg.pTag); } else { - free(pCreateTbReq->ntbCfg.pSchema); + taosMemoryFree(pCreateTbReq->ntbCfg.pSchema); + taosMemoryFree(pCreateTbReq->ntbCfg.pBSmaCols); + taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam); } } @@ -120,7 +128,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { tSerializeSVCreateTbBatchRsp(msg, contLen, &vCreateTbBatchRsp); taosArrayDestroy(vCreateTbBatchRsp.rspList); - *pRsp = calloc(1, sizeof(SRpcMsg)); + *pRsp = taosMemoryCalloc(1, sizeof(SRpcMsg)); (*pRsp)->msgType = TDMT_VND_CREATE_TABLE_RSP; (*pRsp)->pCont = msg; (*pRsp)->contLen = contLen; @@ -133,9 +141,12 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vAlterTbReq = {0}; vTrace("vgId:%d, process alter stb req", pVnode->vgId); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq); - free(vAlterTbReq.stbCfg.pSchema); - free(vAlterTbReq.stbCfg.pTagSchema); - free(vAlterTbReq.name); + taosMemoryFree(vAlterTbReq.stbCfg.pSchema); + taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema); + taosMemoryFree(vAlterTbReq.stbCfg.pBSmaCols); + taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); + taosMemoryFree(vAlterTbReq.dbFName); + taosMemoryFree(vAlterTbReq.name); break; } case TDMT_VND_DROP_STB: @@ -176,7 +187,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // record current timezone of server side - tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezone, TD_TIMEZONE_LEN); + tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) { // TODO: handle error diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp index 4f15185254..627dbc6f18 100644 --- a/source/dnode/vnode/test/tqMetaTest.cpp +++ b/source/dnode/vnode/test/tqMetaTest.cpp @@ -12,7 +12,7 @@ struct Foo { int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { Foo* foo = (Foo*)pObj; if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { - *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); + *ppHead = (STqSerializedHead*)taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); } *(int32_t*)(*ppHead)->content = foo->a; @@ -21,14 +21,14 @@ int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { if (*ppObj == NULL) { - *ppObj = realloc(*ppObj, sizeof(int32_t)); + *ppObj = taosMemoryRealloc(*ppObj, sizeof(int32_t)); } Foo* pFoo = *(Foo**)ppObj; pFoo->a = *(int32_t*)pHead->content; return NULL; } -void FooDeleter(void* pObj) { free(pObj); } +void FooDeleter(void* pObj) { taosMemoryFree(pObj); } class TqMetaUpdateAppendTest : public ::testing::Test { protected: @@ -58,7 +58,7 @@ TEST_F(TqMetaUpdateAppendTest, copyPutTest) { } TEST_F(TqMetaUpdateAppendTest, persistTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 2; tqHandleMovePut(pMeta, 1, pFoo); Foo* pBar = (Foo*)tqHandleGet(pMeta, 1); @@ -82,7 +82,7 @@ TEST_F(TqMetaUpdateAppendTest, persistTest) { } TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); @@ -91,7 +91,7 @@ TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { } TEST_F(TqMetaUpdateAppendTest, abortTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); @@ -104,7 +104,7 @@ TEST_F(TqMetaUpdateAppendTest, abortTest) { } TEST_F(TqMetaUpdateAppendTest, deleteTest) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); @@ -135,12 +135,12 @@ TEST_F(TqMetaUpdateAppendTest, deleteTest) { } TEST_F(TqMetaUpdateAppendTest, intxnPersist) { - Foo* pFoo = (Foo*)malloc(sizeof(Foo)); + Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); tqHandleCommit(pMeta, 1); - Foo* pBar = (Foo*)malloc(sizeof(Foo)); + Foo* pBar = (Foo*)taosMemoryMalloc(sizeof(Foo)); pBar->a = 4; tqHandleMovePut(pMeta, 1, pBar); diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index c13ae83150..29a4b7f552 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -43,7 +43,7 @@ TEST(testCase, unionEncodeDecodeTest) { }; }; col_id_t nBSmaCols; - col_id_t* pBSmaCols; + col_id_t *pBSmaCols; } SUnionTest; SUnionTest sut = {0}; @@ -51,13 +51,13 @@ TEST(testCase, unionEncodeDecodeTest) { sut.type = 1; sut.nBSmaCols = 2; - sut.pBSmaCols = (col_id_t*)malloc(sut.nBSmaCols * sizeof(col_id_t)); + sut.pBSmaCols = (col_id_t *)taosMemoryMalloc(sut.nBSmaCols * sizeof(col_id_t)); for (col_id_t i = 0; i < sut.nBSmaCols; ++i) { sut.pBSmaCols[i] = i + 100; } - void* buf = malloc(1024); - void * pBuf = buf; + void *buf = taosMemoryMalloc(1024); + void *pBuf = buf; int32_t tlen = 0; tlen += taosEncodeFixedU8(&buf, sut.info); tlen += taosEncodeFixedI16(&buf, sut.nBSmaCols); @@ -68,9 +68,9 @@ TEST(testCase, unionEncodeDecodeTest) { SUnionTest dut = {0}; pBuf = taosDecodeFixedU8(pBuf, &dut.info); pBuf = taosDecodeFixedI16(pBuf, &dut.nBSmaCols); - if(dut.nBSmaCols > 0) { - dut.pBSmaCols = (col_id_t*)malloc(dut.nBSmaCols * sizeof(col_id_t)); - for(col_id_t i=0; i < dut.nBSmaCols; ++i) { + if (dut.nBSmaCols > 0) { + dut.pBSmaCols = (col_id_t *)taosMemoryMalloc(dut.nBSmaCols * sizeof(col_id_t)); + for (col_id_t i = 0; i < dut.nBSmaCols; ++i) { pBuf = taosDecodeFixedI16(pBuf, dut.pBSmaCols + i); } } else { @@ -83,9 +83,9 @@ TEST(testCase, unionEncodeDecodeTest) { ASSERT_EQ(sut.rollup, dut.rollup); ASSERT_EQ(sut.type, dut.type); ASSERT_EQ(sut.nBSmaCols, dut.nBSmaCols); - for (col_id_t i = 0; i< sut.nBSmaCols; ++i) { - ASSERT_EQ(*(col_id_t*)(sut.pBSmaCols + i), sut.pBSmaCols[i]); - ASSERT_EQ(*(col_id_t*)(sut.pBSmaCols + i), dut.pBSmaCols[i]); + for (col_id_t i = 0; i < sut.nBSmaCols; ++i) { + ASSERT_EQ(*(col_id_t *)(sut.pBSmaCols + i), sut.pBSmaCols[i]); + ASSERT_EQ(*(col_id_t *)(sut.pBSmaCols + i), dut.pBSmaCols[i]); } } #if 1 @@ -105,7 +105,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma}; uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper); - void *buf = calloc(1, bufLen); + void *buf = taosMemoryCalloc(1, bufLen); ASSERT_NE(buf, nullptr); STSmaWrapper *pSW = (STSmaWrapper *)buf; @@ -115,7 +115,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { // decode STSmaWrapper dstTSmaWrapper = {0}; - void * result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper); + void *result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper); ASSERT_NE(result, nullptr); ASSERT_EQ(tSmaWrapper.number, dstTSmaWrapper.number); @@ -140,7 +140,7 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { } // resource release - tfree(pSW); + taosMemoryFreeClear(pSW); tdDestroyTSma(&tSma); tdDestroyTSmaWrapper(&dstTSmaWrapper); } @@ -148,12 +148,12 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { #if 1 TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { - const char * smaIndexName1 = "sma_index_test_1"; - const char * smaIndexName2 = "sma_index_test_2"; + const char *smaIndexName1 = "sma_index_test_1"; + const char *smaIndexName2 = "sma_index_test_2"; int8_t timezone = 8; - const char * expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; - const char * tagsFilter = "I'm tags filter"; - const char * smaTestDir = "./smaTest"; + const char *expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; + const char *tagsFilter = "I'm tags filter"; + const char *smaTestDir = "./smaTest"; const tb_uid_t tbUid = 1234567890; const int64_t indexUid1 = 2000000001; const int64_t indexUid2 = 2000000002; @@ -171,17 +171,17 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { tSma.tableUid = tbUid; tSma.exprLen = strlen(expr); - tSma.expr = (char *)calloc(1, tSma.exprLen + 1); + tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1); ASSERT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); - tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); + tSma.tagsFilter = (char *)taosMemoryCalloc(tSma.tagsFilterLen + 1, 1); ASSERT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); - SMeta * pMeta = NULL; - STSma * pSmaCfg = &tSma; + SMeta *pMeta = NULL; + STSma *pSmaCfg = &tSma; const SMetaCfg *pMetaCfg = &defaultMetaOptions; taosRemoveDir(smaTestDir); @@ -213,7 +213,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); ASSERT_EQ(qSmaCfg->tableUid, tSma.tableUid); tdDestroyTSma(qSmaCfg); - tfree(qSmaCfg); + taosMemoryFreeClear(qSmaCfg); qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2); assert(qSmaCfg != NULL); @@ -224,7 +224,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); ASSERT_EQ(qSmaCfg->interval, tSma.interval); tdDestroyTSma(qSmaCfg); - tfree(qSmaCfg); + taosMemoryFreeClear(qSmaCfg); // get index name by table uid SMSmaCursor *pSmaCur = metaOpenSmaCursor(pMeta, tbUid); @@ -259,7 +259,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { ASSERT_EQ((pSW->tSma + 1)->tableUid, tbUid); tdDestroyTSmaWrapper(pSW); - tfree(pSW); + taosMemoryFreeClear(pSW); // get all sma table uids SArray *pUids = metaGetSmaTbUids(pMeta, false); @@ -280,14 +280,14 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { } #endif -#if 1 +#if 0 TEST(testCase, tSma_Data_Insert_Query_Test) { // step 1: prepare meta - const char * smaIndexName1 = "sma_index_test_1"; + const char *smaIndexName1 = "sma_index_test_1"; const int8_t timezone = 8; - const char * expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; - const char * tagsFilter = "where tags.location='Beijing' and tags.district='ChaoYang'"; - const char * smaTestDir = "./smaTest"; + const char *expr = "select count(a,b, top 20), from table interval 1d, sliding 1h;"; + const char *tagsFilter = "where tags.location='Beijing' and tags.district='ChaoYang'"; + const char *smaTestDir = "./smaTest"; const tb_uid_t tbUid = 1234567890; const int64_t indexUid1 = 2000000001; const int64_t interval1 = 1; @@ -302,24 +302,24 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { tSma.intervalUnit = TIME_UNIT_DAY; tSma.interval = 1; tSma.slidingUnit = TIME_UNIT_HOUR; - tSma.sliding = 0; + tSma.sliding = 1; // sliding = interval when it's convert window tSma.indexUid = indexUid1; tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); tSma.timezoneInt = timezone; tSma.tableUid = tbUid; tSma.exprLen = strlen(expr); - tSma.expr = (char *)calloc(1, tSma.exprLen + 1); + tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1); ASSERT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); - tSma.tagsFilter = (char *)calloc(1, tSma.tagsFilterLen + 1); + tSma.tagsFilter = (char *)taosMemoryCalloc(1, tSma.tagsFilterLen + 1); ASSERT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); - SMeta * pMeta = NULL; - STSma * pSmaCfg = &tSma; + SMeta *pMeta = NULL; + STSma *pSmaCfg = &tSma; const SMetaCfg *pMetaCfg = &defaultMetaOptions; taosRemoveDir(smaTestDir); @@ -331,8 +331,8 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { // step 2: insert data STSmaDataWrapper *pSmaData = NULL; - STsdb * pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); - STsdbCfg * pCfg = &pTsdb->config; + STsdb *pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); + STsdbCfg *pCfg = &pTsdb->config; pTsdb->pMeta = pMeta; pTsdb->vgId = 2; @@ -367,15 +367,49 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); ASSERT_NE(pTsdb->pTfs, nullptr); - char *msg = (char *)calloc(1, 100); - ASSERT_NE(msg, nullptr); - ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, msg), 0); + // generate SSubmitReq msg and update expired window + int16_t schemaVer = 0; + uint32_t mockRowLen = sizeof(STSRow); + uint32_t mockRowNum = 2; + uint32_t mockBlkNum = 2; + uint32_t msgLen = sizeof(SSubmitReq) + mockBlkNum * sizeof(SSubmitBlk) + mockBlkNum * mockRowNum * mockRowLen; + + SSubmitReq *pMsg = (SSubmitReq *)taosMemoryCalloc(1, msgLen); + ASSERT_NE(pMsg, nullptr); + pMsg->version = htobe64(schemaVer); + pMsg->numOfBlocks = htonl(mockBlkNum); + pMsg->length = htonl(msgLen); + + SSubmitBlk *pBlk = NULL; + STSRow *pRow = NULL; + TSKEY now = taosGetTimestamp(pTsdb->config.precision); + + for (uint32_t b = 0; b < mockBlkNum; ++b) { + pBlk = (SSubmitBlk *)POINTER_SHIFT(pMsg, sizeof(SSubmitReq) + b * (sizeof(SSubmitBlk) + mockRowNum * mockRowLen)); + pBlk->uid = htobe64(tbUid); + pBlk->suid = htobe64(tbUid); + pBlk->sversion = htonl(schemaVer); + pBlk->padding = htonl(0); + pBlk->schemaLen = htonl(0); + pBlk->numOfRows = htons(mockRowNum); + pBlk->dataLen = htonl(mockRowNum * mockRowLen); + for (uint32_t r = 0; r < mockRowNum; ++r) { + pRow = (STSRow *)POINTER_SHIFT(pBlk, sizeof(SSubmitBlk) + r * mockRowLen); + pRow->len = mockRowLen; + pRow->ts = now + b * 1000 + r * 1000; + pRow->sver = schemaVer; + } + } + + ASSERT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS); + + ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, (const char *)pMsg), 0); // init int32_t allocCnt = 0; int32_t allocStep = 16384; int32_t buffer = 1024; - void * buf = NULL; + void *buf = NULL; ASSERT_EQ(tsdbMakeRoom(&buf, allocStep), 0); int32_t bufSize = taosTSizeof(buf); int32_t numOfTables = 10; @@ -443,7 +477,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt); // release data - tfree(msg); + taosMemoryFreeClear(pMsg); taosTZfree(buf); // release meta tdDestroyTSma(&tSma); @@ -451,6 +485,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { tsdbClose(pTsdb); metaClose(pMeta); } + #endif #pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index ea8195bfd1..b448a43dcb 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(scheduler) add_subdirectory(cache) add_subdirectory(catalog) add_subdirectory(executor) +add_subdirectory(stream) add_subdirectory(planner) add_subdirectory(function) add_subdirectory(qcom) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 8f67482650..e87fdba71d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -172,7 +172,7 @@ void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { int32_t colNum = c->numOfColumns + c->numOfTags; for (int32_t i = 0; i < colNum; ++i) { SSchema *s = &p->schema[i]; - ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); + ctgDebug("[%d] name:%s, type:%d, colId:%" PRIi16 ", bytes:%d", i, s->name, s->type, s->colId, s->bytes); } } @@ -242,7 +242,7 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { } } - tfree(mgmt->slots); + taosMemoryFreeClear(mgmt->slots); } @@ -272,7 +272,7 @@ void ctgFreeVgInfo(SDBVgInfo *vgInfo) { vgInfo->vgHash = NULL; } - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } void ctgFreeDbCache(SCtgDBCache *dbCache) { @@ -307,7 +307,7 @@ void ctgFreeHandle(SCatalog* pCtg) { taosHashCleanup(pCtg->dbCache); } - free(pCtg); + taosMemoryFree(pCtg); } @@ -338,14 +338,14 @@ void ctgPopAction(SCtgMetaAction **action) { CTG_QUEUE_SUB(); - tfree(orig); + taosMemoryFreeClear(orig); *action = &node->action; } int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { - SCtgQNode *node = calloc(1, sizeof(SCtgQNode)); + SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); CTG_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -378,7 +378,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { int32_t code = 0; SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; - SCtgRemoveDBMsg *msg = malloc(sizeof(SCtgRemoveDBMsg)); + SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -401,15 +401,15 @@ int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) _return: - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } -int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid) { +int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) { int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB}; - SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg)); + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; + SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -429,16 +429,16 @@ int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId _return: - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } -int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName) { +int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL}; - SCtgRemoveTblMsg *msg = malloc(sizeof(SCtgRemoveTblMsg)); + SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; + SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -457,14 +457,14 @@ int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId _return: - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } int32_t ctgPushUpdateVgMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) { int32_t code = 0; SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; - SCtgUpdateVgMsg *msg = malloc(sizeof(SCtgUpdateVgMsg)); + SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); ctgFreeVgInfo(dbInfo); @@ -490,14 +490,14 @@ int32_t ctgPushUpdateVgMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t d _return: ctgFreeVgInfo(dbInfo); - tfree(action.data); + taosMemoryFreeClear(action.data); CTG_RET(code); } int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) { int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL}; - SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg)); + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; + SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -519,7 +519,7 @@ int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, boo _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -818,7 +818,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); - tfree(*pTableMeta); + taosMemoryFreeClear(*pTableMeta); *exist = 0; return TSDB_CODE_SUCCESS; } @@ -826,13 +826,13 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if ((*stbMeta)->suid != tbMeta->suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); - tfree(*pTableMeta); + taosMemoryFreeClear(*pTableMeta); ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = CTG_META_SIZE(*stbMeta); - *pTableMeta = realloc(*pTableMeta, metaSize); + *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); @@ -1161,7 +1161,7 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; - mgmt->slots = calloc(1, msgSize); + mgmt->slots = taosMemoryCalloc(1, msgSize); if (NULL == mgmt->slots) { qError("calloc %d failed", (int32_t)msgSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1305,7 +1305,7 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_ } size_t msize = metaNum * size; - *res = malloc(msize); + *res = taosMemoryMalloc(msize); if (NULL == *res) { qError("malloc %d failed", (int32_t)msize); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); @@ -1631,7 +1631,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui } int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { - *dst = malloc(sizeof(SDBVgInfo)); + *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); if (NULL == *dst) { qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1643,7 +1643,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == (*dst)->vgHash) { qError("taosHashInit %d failed", (int32_t)hashSize); - tfree(*dst); + taosMemoryFreeClear(*dst); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1656,7 +1656,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); taosHashCancelIterate(src->vgHash, pIter); taosHashCleanup((*dst)->vgHash); - tfree(*dst); + taosMemoryFreeClear(*dst); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1703,7 +1703,7 @@ int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const _return: - tfree(*pInfo); + taosMemoryFreeClear(*pInfo); *pInfo = DbOut.dbVgroup; CTG_RET(code); @@ -1748,7 +1748,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, c int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { - *pOutput = malloc(sizeof(STableMetaOutput)); + *pOutput = taosMemoryMalloc(sizeof(STableMetaOutput)); if (NULL == *pOutput) { qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1758,10 +1758,10 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) if (output->tbMeta) { int32_t metaSize = CTG_META_SIZE(output->tbMeta); - (*pOutput)->tbMeta = malloc(metaSize); + (*pOutput)->tbMeta = taosMemoryMalloc(metaSize); if (NULL == (*pOutput)->tbMeta) { qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); - tfree(*pOutput); + taosMemoryFreeClear(*pOutput); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1786,7 +1786,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } STableMetaOutput moutput = {0}; - STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); + STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); if (NULL == output) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -1814,7 +1814,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { @@ -1830,11 +1830,11 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SET_META_TYPE_NULL(output->metaType); } - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); SET_META_TYPE_CTABLE(output->metaType); } @@ -1843,6 +1843,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (CTG_IS_META_NULL(output->metaType)) { ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pTableName)); + catalogRemoveTableMeta(pCtg, pTableName); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1862,8 +1863,8 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, _return: - tfree(output->tbMeta); - tfree(output); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); CTG_RET(code); } @@ -1895,7 +1896,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons tbType = (*pTableMeta)->tableType; suid = (*pTableMeta)->suid; - tfree(*pTableMeta); + taosMemoryFreeClear(*pTableMeta); } if (CTG_FLAG_IS_UNKNOWN_STB(flag)) { @@ -1920,7 +1921,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons if ((!CTG_IS_META_CTABLE(output->metaType)) || output->tbMeta) { ctgError("invalid metaType:%d", output->metaType); - tfree(output->tbMeta); + taosMemoryFreeClear(output->tbMeta); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1951,13 +1952,13 @@ _return: } if (TSDB_SUPER_TABLE == tbType) { - ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid); + ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid, false); } else { - ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname); + ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, false); } } - tfree(output); + taosMemoryFreeClear(output); if (*pTableMeta) { ctgDebug("tbmeta returned, tbName:%s, tbType:%d", pTableName->tname, (*pTableMeta)->tableType); @@ -1978,7 +1979,7 @@ int32_t ctgActUpdateVg(SCtgMetaAction *action) { _return: ctgFreeVgInfo(msg->dbInfo); - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2003,7 +2004,7 @@ int32_t ctgActRemoveDB(SCtgMetaAction *action) { _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2045,11 +2046,11 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { _return: if (output) { - tfree(output->tbMeta); - tfree(output); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); } - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2092,7 +2093,7 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2125,7 +2126,7 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { _return: - tfree(msg); + taosMemoryFreeClear(msg); CTG_RET(code); } @@ -2248,11 +2249,11 @@ _return: ctgReleaseDBCache(pCtg, dbCache); } - tfree(tbMeta); + taosMemoryFreeClear(tbMeta); if (vgInfo) { taosHashCleanup(vgInfo->vgHash); - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } if (vgList) { @@ -2308,7 +2309,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { tsem_init(&gCtgMgmt.queue.reqSem, 0, 0); tsem_init(&gCtgMgmt.queue.rspSem, 0, 0); - gCtgMgmt.queue.head = calloc(1, sizeof(SCtgQNode)); + gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == gCtgMgmt.queue.head) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -2342,7 +2343,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { return TSDB_CODE_SUCCESS; } - clusterCtg = calloc(1, sizeof(SCatalog)); + clusterCtg = taosMemoryCalloc(1, sizeof(SCatalog)); if (NULL == clusterCtg) { qError("calloc %d failed", (int32_t)sizeof(SCatalog)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); @@ -2483,7 +2484,7 @@ _return: if (vgInfo) { taosHashCleanup(vgInfo->vgHash); - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } CTG_API_LEAVE(code); @@ -2534,7 +2535,7 @@ int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, } -int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { +int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { CTG_API_ENTER(); int32_t code = 0; @@ -2561,15 +2562,15 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { tNameGetFullDbName(pTableName, dbFName); if (TSDB_SUPER_TABLE == tblMeta->tableType) { - CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid)); + CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid, true)); } else { - CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname)); + CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, true)); } _return: - tfree(tblMeta); + taosMemoryFreeClear(tblMeta); CTG_API_LEAVE(code); } @@ -2588,7 +2589,7 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid)); + CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid, true)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); @@ -2620,7 +2621,7 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); + STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); if (NULL == output) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); @@ -2643,8 +2644,8 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { _return: - tfree(output->tbMeta); - tfree(output); + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); CTG_API_LEAVE(code); } @@ -2739,7 +2740,7 @@ _return: if (vgInfo) { taosHashCleanup(vgInfo->vgHash); - tfree(vgInfo); + taosMemoryFreeClear(vgInfo); } CTG_API_LEAVE(code); @@ -2777,7 +2778,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); - tfree(pTableMeta); + taosMemoryFreeClear(pTableMeta); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -2795,7 +2796,7 @@ _return: int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); for (int32_t i = 0; i < aSize; ++i) { STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i); - tfree(pMeta); + taosMemoryFreeClear(pMeta); } taosArrayDestroy(pRsp->pTableMeta); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index aff2c2edcd..e62819b078 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -176,7 +176,7 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { output->ctbMeta.uid = 3; output->ctbMeta.suid = 2; - output->tbMeta = (STableMeta *)calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum)); + output->tbMeta = (STableMeta *)taosMemoryCalloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum)); output->tbMeta->vgId = 9; output->tbMeta->tableType = TSDB_SUPER_TABLE; output->tbMeta->uid = 2; @@ -212,7 +212,7 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; - SDBVgInfo *dbVgroup = (SDBVgInfo *)calloc(1, sizeof(SDBVgInfo)); + SDBVgInfo *dbVgroup = (SDBVgInfo *)taosMemoryCalloc(1, sizeof(SDBVgInfo)); dbVgroup->vgVersion = vgVersion++; @@ -257,7 +257,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { rspMsg->tuid = ctgTestSuid + 1; rspMsg->vgId = 1; - rspMsg->pSchemas = (SSchema *)calloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); + rspMsg->pSchemas = (SSchema *)taosMemoryCalloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); SSchema *s = NULL; s = &rspMsg->pSchemas[0]; @@ -335,7 +335,7 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * metaRsp.suid = 0; metaRsp.tuid = ctgTestNormalTblUid++; metaRsp.vgId = 8; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -381,7 +381,7 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.suid = 0x0000000000000002; metaRsp.tuid = 0x0000000000000003; metaRsp.vgId = 9; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -428,7 +428,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.suid = ctgTestSuid; metaRsp.tuid = ctgTestSuid++; metaRsp.vgId = 0; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -477,7 +477,7 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp metaRsp.suid = ctgTestSuid + idx; metaRsp.tuid = ctgTestSuid + idx; metaRsp.vgId = 0; - metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; @@ -798,7 +798,7 @@ void *ctgTestGetCtableMetaThread(void *param) { assert(0); } - tfree(tbMeta); + taosMemoryFreeClear(tbMeta); if (ctgTestEnableSleep) { taosUsleep(taosRand() % 5); @@ -824,10 +824,10 @@ void *ctgTestSetCtableMetaThread(void *param) { action.act = CTG_ACT_UPDATE_TBL; while (!ctgTestStop) { - output = (STableMetaOutput *)malloc(sizeof(STableMetaOutput)); + output = (STableMetaOutput *)taosMemoryMalloc(sizeof(STableMetaOutput)); ctgTestBuildCTableMetaOutput(output); - SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)malloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); msg->pCtg = pCtg; msg->output = output; action.data = msg; @@ -933,7 +933,7 @@ TEST(tableMeta, normalTable) { if (dbNum) { printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } else { printf("no expired db\n"); @@ -941,7 +941,7 @@ TEST(tableMeta, normalTable) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - free(stb); + taosMemoryFree(stb); stb = NULL; } else { printf("no expired stb\n"); @@ -1015,7 +1015,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); strcpy(n.tname, ctgTestSTablename); code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); @@ -1042,7 +1042,7 @@ TEST(tableMeta, childTableCase) { if (dbNum) { printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } else { printf("no expired db\n"); @@ -1050,7 +1050,7 @@ TEST(tableMeta, childTableCase) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - free(stb); + taosMemoryFree(stb); stb = NULL; } else { printf("no expired stb\n"); @@ -1164,7 +1164,7 @@ TEST(tableMeta, superTableCase) { if (dbNum) { printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } else { printf("no expired db\n"); @@ -1173,7 +1173,7 @@ TEST(tableMeta, superTableCase) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - free(stb); + taosMemoryFree(stb); stb = NULL; } else { printf("no expired stb\n"); @@ -1307,14 +1307,14 @@ TEST(tableMeta, updateStbMeta) { } - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); STableMetaRsp rsp = {0}; ctgTestBuildSTableMetaRsp(&rsp); code = catalogUpdateSTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); - tfree(rsp.pSchemas); + taosMemoryFreeClear(rsp.pSchemas); while (true) { uint64_t n = 0; @@ -1345,7 +1345,7 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.precision, 1 + 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat)); @@ -1407,7 +1407,7 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1424,7 +1424,7 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1486,7 +1486,7 @@ TEST(refreshGetMeta, normal2notexist) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1560,7 +1560,7 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, 0); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1576,7 +1576,7 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1645,7 +1645,7 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1662,7 +1662,7 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1730,7 +1730,7 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1748,7 +1748,7 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -1816,7 +1816,7 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { taosMsleep(50); @@ -1835,7 +1835,7 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - tfree(tableMeta); + taosMemoryFreeClear(tableMeta); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -2275,7 +2275,7 @@ TEST(rentTest, allRent) { printf("%d - expired dbNum:%d\n", i, num); if (dbs) { printf("%d - expired dbId:%" PRId64 ", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion); - free(dbs); + taosMemoryFree(dbs); dbs = NULL; } @@ -2287,7 +2287,7 @@ TEST(rentTest, allRent) { printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } - free(stable); + taosMemoryFree(stable); stable = NULL; } printf("*************************************************\n"); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c582873315..468e2c5431 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -415,6 +415,7 @@ typedef struct STableScanInfo { int32_t* rowCellInfoOffset; SExprInfo* pExpr; SSDataBlock block; + SArray* pColMatchInfo; int32_t numOfOutput; int64_t elapsedTime; int32_t prevGroupId; // previous table group id @@ -449,6 +450,8 @@ typedef struct SSysTableScanInfo { SEpSet epSet; tsem_t ready; + int32_t accountId; + bool showRewrite; SNode* pCondition; // db_name filter condition, to discard data that are not in current database void *pCur; // cursor for iterate the local table meta store. SArray *scanCols; // SArray scan column id list @@ -646,8 +649,8 @@ typedef struct SDistinctOperatorInfo { } SDistinctOperatorInfo; SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, - int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t repeatTime, + int32_t reverseTime, SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); @@ -655,7 +658,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, - SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo); + SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId); SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 1c2d9424d3..34894c235b 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -142,7 +142,7 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockDataGetSerialMetaSize(pInput->pData) + ceil(blockDataGetSerialRowSize(pInput->pData) * pInput->pData->info.rows); - pBuf->pData = malloc(pBuf->allocSize); + pBuf->pData = taosMemoryMalloc(pBuf->allocSize); if (pBuf->pData == NULL) { qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); } @@ -215,7 +215,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; pOutput->compressed = pEntry->compressed; - tfree(pDispatcher->nextOutput.pData); // todo persistent + taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); taosThreadMutexLock(&pDispatcher->mutex); pOutput->queryEnd = pDispatcher->queryEnd; @@ -227,11 +227,11 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - tfree(pDispatcher->nextOutput.pData); + taosMemoryFreeClear(pDispatcher->nextOutput.pData); while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); - tfree(pBuf->pData); + taosMemoryFreeClear(pBuf->pData); taosFreeQitem(pBuf); } taosCloseQueue(pDispatcher->pDataBlocks); @@ -239,7 +239,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { } int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { - SDataDispatchHandle* dispatcher = calloc(1, sizeof(SDataDispatchHandle)); + SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a04a10ef95..f0cffafca2 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -58,8 +58,8 @@ int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) { pResultRowInfo->curPos = -1; pResultRowInfo->capacity = size; - pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES); - pResultRowInfo->pPosition = calloc(pResultRowInfo->capacity, sizeof(SResultRowPosition)); + pResultRowInfo->pResult = taosMemoryCalloc(pResultRowInfo->capacity, POINTER_BYTES); + pResultRowInfo->pPosition = taosMemoryCalloc(pResultRowInfo->capacity, sizeof(SResultRowPosition)); if (pResultRowInfo->pResult == NULL || pResultRowInfo->pPosition == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -79,11 +79,11 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { for(int32_t i = 0; i < pResultRowInfo->size; ++i) { if (pResultRowInfo->pResult[i]) { - tfree(pResultRowInfo->pResult[i]->key); + taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); } } - tfree(pResultRowInfo->pResult); + taosMemoryFreeClear(pResultRowInfo->pResult); } void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { @@ -163,7 +163,7 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { pResultRow->offset = -1; pResultRow->closed = false; - tfree(pResultRow->key); + taosMemoryFreeClear(pResultRow->key); pResultRow->win = TSWINDOW_INITIALIZER; } @@ -190,7 +190,7 @@ SResultRow* getNewResultRow(SResultRowPool* p) { void* ptr = NULL; if (p->position.pos == 0) { - ptr = calloc(1, p->blockSize); + ptr = taosMemoryCalloc(1, p->blockSize); taosArrayPush(p->pData, &ptr); } else { @@ -403,8 +403,8 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); } - posList = calloc(size, sizeof(int32_t)); - pTableQueryInfoList = malloc(POINTER_BYTES * size); + posList = taosMemoryCalloc(size, sizeof(int32_t)); + pTableQueryInfoList = taosMemoryMalloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { // qError("QInfo:%"PRIu64" failed alloc memory", GET_TASKID(pRuntimeEnv)); @@ -483,9 +483,9 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv // pGroupResInfo->currentGroup, endt - startt); _end: - tfree(pTableQueryInfoList); - tfree(posList); - tfree(pTree); + taosMemoryFreeClear(pTableQueryInfoList); + taosMemoryFreeClear(posList); + taosMemoryFreeClear(pTree); return code; } @@ -529,7 +529,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // // // compress extra bytes // size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize; -// char* tmp = malloc(x + 2); +// char* tmp = taosMemoryMalloc(x + 2); // // bool comp = false; // int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0); @@ -547,7 +547,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // } else { // tbufWriteBinary(bw, p, len); // } -// tfree(tmp); +// taosMemoryFreeClear(tmp); //} //void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { @@ -570,7 +570,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // // char* outputBuf = NULL; // if (comp) { -// outputBuf = malloc(originalLen); +// outputBuf = taosMemoryMalloc(originalLen); // // size_t actualLen = compLen; // const char* compStr = tbufReadBinary(&br, &actualLen); @@ -584,7 +584,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun // // pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); // if (comp) { -// tfree(outputBuf); +// taosMemoryFreeClear(outputBuf); // } //} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e89bc5df0e..26422fa618 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -97,6 +97,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { pMsg->contentLen = pMsg->contentLen; #endif + qDebugL("stream task string %s", (const char*)msg); + struct SSubplan* plan = NULL; int32_t code = qStringToSubplan(msg, &plan); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index cea9b3ce11..f5124665d5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -66,6 +66,11 @@ typedef enum SResultTsInterpType { RESULT_ROW_END_INTERP = 2, } SResultTsInterpType; +typedef struct SColMatchInfo { + int32_t colId; + int32_t targetSlotId; +} SColMatchInfo; + #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = taosRand(); @@ -73,7 +78,7 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { if (v % 1000 <= 0) { return NULL; } else { - return malloc(__size); + return taosMemoryMalloc(__size); } } @@ -82,7 +87,7 @@ static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { if (v % 1000 <= 0) { return NULL; } else { - return calloc(num, __size); + return taosMemoryCalloc(num, __size); } } @@ -91,7 +96,7 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { if (v % 5 <= 1) { return NULL; } else { - return realloc(p, __size); + return taosMemoryRealloc(p, __size); } } @@ -320,7 +325,7 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, STaskRuntimeEn SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { const static int32_t minSize = 8; - SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = numOfOutput; res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfOutput; ++i) { @@ -330,7 +335,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO idata.info.colId = pExpr[i].base.resSchema.colId; int32_t size = TMAX(idata.info.bytes * numOfRows, minSize); - idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform + idata.pData = taosMemoryCalloc(1, size); // at least to hold a pointer on x64 platform taosArrayPush(res->pDataBlock, &idata); } @@ -340,7 +345,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); - SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->info.numOfCols = numOfCols; pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); @@ -423,12 +428,12 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) newCapacity += 4; } - char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + char *t = taosMemoryRealloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pResultRowInfo->pPosition = realloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); + pResultRowInfo->pPosition = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); pResultRowInfo->pResult = (SResultRow **)t; int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; @@ -1857,7 +1862,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { if (IS_VAR_DATA_TYPE(type)) { if (pResultRow->key == NULL) { - pResultRow->key = malloc(varDataTLen(pData)); + pResultRow->key = taosMemoryMalloc(varDataTLen(pData)); varDataCopy(pResultRow->key, pData); } else { assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); @@ -1976,7 +1981,7 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { int16_t tagLen = 0; SqlFunctionCtx* p = NULL; - SqlFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + SqlFunctionCtx** pTagCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); if (pTagCtx == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -2002,7 +2007,7 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { p->subsidiaryRes.numOfCols = num; p->subsidiaryRes.bufLen = tagLen; } else { - tfree(pTagCtx); + taosMemoryFreeClear(pTagCtx); } return TSDB_CODE_SUCCESS; @@ -2012,14 +2017,14 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI int32_t** rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } - *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + *rowCellInfoOffset = taosMemoryCalloc(numOfOutput, sizeof(int32_t)); if (*rowCellInfoOffset == 0) { - tfree(pFuncCtx); + taosMemoryFreeClear(pFuncCtx); return NULL; } @@ -2110,14 +2115,14 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI } static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset) { - SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } - *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + *rowCellInfoOffset = taosMemoryCalloc(numOfOutput, sizeof(int32_t)); if (*rowCellInfoOffset == 0) { - tfree(pFuncCtx); + taosMemoryFreeClear(pFuncCtx); return NULL; } @@ -2139,8 +2144,8 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num } pCtx->input.numOfInputCols = pFunct->numOfParams; - pCtx->input.pData = calloc(pFunct->numOfParams, POINTER_BYTES); - pCtx->input.pColumnDataAgg = calloc(pFunct->numOfParams, POINTER_BYTES); + pCtx->input.pData = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); + pCtx->input.pColumnDataAgg = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); pCtx->ptsOutputBuf = NULL; pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; @@ -2216,10 +2221,10 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } taosVariantDestroy(&pCtx[i].tag); - tfree(pCtx[i].subsidiaryRes.pCtx); + taosMemoryFreeClear(pCtx[i].subsidiaryRes.pCtx); } - tfree(pCtx); + taosMemoryFreeClear(pCtx); return NULL; } @@ -2232,12 +2237,12 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); + pRuntimeEnv->keyBuf = taosMemoryMalloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); // pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); - pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); - pRuntimeEnv->tagVal = malloc(pQueryAttr->tagLen); + pRuntimeEnv->prevRow = taosMemoryMalloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); + pRuntimeEnv->tagVal = taosMemoryMalloc(pQueryAttr->tagLen); // NOTE: pTableCheckInfo need to update the query time range and the lastKey info pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -2269,10 +2274,10 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT _clean: //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); - tfree(pRuntimeEnv->pResultRowHashTable); - tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->prevRow); - tfree(pRuntimeEnv->tagVal); + taosMemoryFreeClear(pRuntimeEnv->pResultRowHashTable); + taosMemoryFreeClear(pRuntimeEnv->keyBuf); + taosMemoryFreeClear(pRuntimeEnv->prevRow); + taosMemoryFreeClear(pRuntimeEnv->tagVal); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -2806,7 +2811,7 @@ void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo SSDataBlock* pBlock, bool ascQuery) { int32_t numOfRows = pBlock->info.rows; - int8_t *p = calloc(numOfRows, sizeof(int8_t)); + int8_t *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); bool all = true; #if 0 if (pRuntimeEnv->pTsBuf != NULL) { @@ -2842,7 +2847,7 @@ void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo doCompactSDataBlock(pBlock, numOfRows, p); } - tfree(p); + taosMemoryFreeClear(p); } void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { @@ -2853,7 +2858,7 @@ void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, if (pRuntimeEnv->pTsBuf != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - p = calloc(numOfRows, sizeof(int8_t)); + p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); TSKEY* k = (TSKEY*) pColInfoData->pData; for (int32_t i = 0; i < numOfRows; ++i) { @@ -2889,7 +2894,7 @@ void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, } } - tfree(p); + taosMemoryFreeClear(p); } static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); @@ -2944,12 +2949,21 @@ int32_t loadDataBlock(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, *status = BLK_DATA_ALL_NEEDED; - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); - if (pBlock->pDataBlock == NULL) { + SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); + if (pCols == NULL) { return terrno; - } else { - return TSDB_CODE_SUCCESS; } + + int32_t numOfCols = pBlock->info.numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pCols, i); + SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); + ASSERT(pColMatchInfo->colId == p->info.colId); + + taosArraySet(pBlock->pDataBlock, pColMatchInfo->targetSlotId, p); + } + + return TSDB_CODE_SUCCESS; } int32_t loadDataBlockOnDemand(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { @@ -3410,7 +3424,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); - char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); + char* p = taosMemoryRealloc(pColInfo->pData, newSize * pColInfo->info.bytes); if (p != NULL) { pColInfo->pData = p; @@ -3598,7 +3612,7 @@ STableQueryInfo *createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow } STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { - STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); + STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(1, sizeof(STableQueryInfo)); // pTableQueryInfo->win = win; pTableQueryInfo->lastKey = win.skey; @@ -3607,7 +3621,7 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { int32_t initialSize = 16; int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); if (code != TSDB_CODE_SUCCESS) { - tfree(pTableQueryInfo); + taosMemoryFreeClear(pTableQueryInfo); return NULL; } @@ -4425,7 +4439,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t assert(p->numOfDownstream == 0); } - p->pDownstream = calloc(1, num * POINTER_BYTES); + p->pDownstream = taosMemoryCalloc(1, num * POINTER_BYTES); if (p->pDownstream == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -4780,7 +4794,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo *pOperator, bool* newgroup) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); int32_t len = (int32_t) tbufTell(&bw); - pColInfo->pData = malloc(len + sizeof(int32_t)); + pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t)); *(int32_t*) pColInfo->pData = len; memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); @@ -4861,8 +4875,8 @@ int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->msgInfo.pData); - tfree(pMsgBody); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + taosMemoryFreeClear(pMsgBody); } void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { @@ -4872,7 +4886,7 @@ void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; if (pMsg->contLen > 0) { - buf.pData = calloc(1, pMsg->contLen); + buf.pData = taosMemoryCalloc(1, pMsg->contLen); if (buf.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; pMsg->code = TSDB_CODE_OUT_OF_MEMORY; @@ -4889,7 +4903,7 @@ void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInfo *pTaskInfo, int32_t sourceIndex) { size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - SResFetchReq* pMsg = calloc(1, sizeof(SResFetchReq)); + SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return pTaskInfo->code; @@ -4907,9 +4921,9 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf pMsg->queryId = htobe64(pTaskInfo->id.queryId); // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - tfree(pMsg); + taosMemoryFreeClear(pMsg); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return pTaskInfo->code; @@ -4947,7 +4961,7 @@ static int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t)*numOfRows); pStart += sizeof(int32_t)*numOfRows; - pColInfoData->pData = malloc(colLen[i]); + pColInfoData->pData = taosMemoryMalloc(colLen[i]); } else { memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); pStart += BitmapLen(numOfRows); @@ -5237,8 +5251,8 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo *pOperator, bool* newgroup) { #if 0 _error: - tfree(pMsg); - tfree(pMsgSendInfo); + taosMemoryFreeClear(pMsg); + taosMemoryFreeClear(pMsgSendInfo); terrno = pTaskInfo->code; return NULL; @@ -5268,12 +5282,12 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { } SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { - SExchangeInfo* pInfo = calloc(1, sizeof(SExchangeInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -5342,14 +5356,14 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock destroyExchangeOperatorInfo(pInfo, numOfSources); } - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { - SSDataBlock* pResBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pResBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pResBlock == NULL) { return NULL; } @@ -5374,25 +5388,33 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { return pResBlock; } -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, + SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } + pInfo->block.pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData idata = {0}; + taosArrayPush(pInfo->block.pDataBlock, &idata); + } + pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->order = order; pInfo->current = 0; pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; @@ -5406,7 +5428,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, } SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->times = 1; @@ -5416,7 +5438,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim pInfo->prevGroupId = -1; pRuntimeEnv->enableGroupData = true; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableSeqScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; pOperator->blockingOptr = false; @@ -5430,7 +5452,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim } SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); @@ -5441,7 +5463,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt infoData.info.colId = 0; taosArrayPush(pInfo->block.pDataBlock, &infoData); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; pOperator->blockingOptr = false; @@ -5454,11 +5476,11 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt } SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { - SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -5467,8 +5489,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SSDataBlock* tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); if (code != 0) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); return NULL; } @@ -5489,7 +5511,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SSDataBlock* } static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t code) { - SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*) param; + SOperatorInfo* operator = (SOperatorInfo *)param; + SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo *)operator->info; if (TSDB_CODE_SUCCESS == code) { pScanResInfo->pRsp = pMsg->pData; @@ -5498,6 +5521,8 @@ static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t pRsp->useconds = htobe64(pRsp->useconds); pRsp->handle = htobe64(pRsp->handle); pRsp->compLen = htonl(pRsp->compLen); + } else { + operator->pTaskInfo->code = code; } tsem_post(&pScanResInfo->ready); @@ -5544,6 +5569,64 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0? NULL:pInfo->pRes; } +EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { + int32_t code = TSDB_CODE_SUCCESS; + ENodeType nType = nodeType(pNode); + + switch (nType) { + case QUERY_NODE_OPERATOR: { + SOperatorNode *node = (SOperatorNode *)pNode; + + if (OP_TYPE_EQUAL == node->opType) { + *(int32_t *)pContext = 1; + return DEAL_RES_CONTINUE; + } + + *(int32_t *)pContext = 0; + + return DEAL_RES_IGNORE_CHILD; + } + case QUERY_NODE_COLUMN: { + if (1 != *(int32_t *)pContext) { + return DEAL_RES_CONTINUE; + } + + SColumnNode *node = (SColumnNode *)pNode; + if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) { + *(int32_t *)pContext = 2; + return DEAL_RES_CONTINUE; + } + + *(int32_t *)pContext = 0; + return DEAL_RES_CONTINUE; + } + case QUERY_NODE_VALUE: { + if (2 != *(int32_t *)pContext) { + return DEAL_RES_CONTINUE; + } + + SValueNode *node = (SValueNode *)pNode; + char *dbName = nodesGetValueFromNode(node); + strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); + *((char *)pContext + varDataLen(dbName)) = 0; + return DEAL_RES_ERROR; // stop walk + } + default: + break; + } + + return DEAL_RES_CONTINUE; +} + + +void getDBNameFromCondition(SNode *pCondition, char *dbName) { + if (NULL == pCondition) { + return; + } + + nodesWalkNode(pCondition, getDBNameFromConditionWalker, dbName); +} + static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { // build message and send to mnode to fetch the content of system tables. SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -5600,20 +5683,25 @@ static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { pInfo->req.type = pInfo->type; strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + if (pInfo->showRewrite) { + char dbName[TSDB_DB_NAME_LEN] = {0}; + getDBNameFromCondition(pInfo->pCondition, dbName); + sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + } int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); - char* buf1 = calloc(1, contLen); + char* buf1 = taosMemoryCalloc(1, contLen); tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } - pMsgSendInfo->param = pInfo; + pMsgSendInfo->param = pOperator; pMsgSendInfo->msgInfo.pData = buf1; pMsgSendInfo->msgInfo.len = contLen; pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; @@ -5623,6 +5711,10 @@ static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); tsem_wait(&pInfo->ready); + if (pTaskInfo->code) { + return NULL; + } + SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; pInfo->req.showId = pRsp->handle; @@ -5644,20 +5736,22 @@ static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { } SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, - SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo) { - SSysTableScanInfo* pInfo = calloc(1, sizeof(SSysTableScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId) { + SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } - pInfo->pRes = pResBlock; - pInfo->capacity = 4096; - pInfo->pCondition = pCondition; - pInfo->scanCols = colList; + pInfo->accountId = accountId; + pInfo->showRewrite = showRewrite; + pInfo->pRes = pResBlock; + pInfo->capacity = 4096; + pInfo->pCondition = pCondition; + pInfo->scanCols = colList; // TODO remove it int32_t tableType = 0; @@ -5832,7 +5926,7 @@ static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param; taosArrayDestroy(pInfo->orderColumnList); pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->prevRow); + taosMemoryFreeClear(pInfo->prevRow); } static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { @@ -5841,7 +5935,7 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { *dst = *src; dst->pExpr = exprdup(src->pExpr); - dst->base.pParam = calloc(src->base.numOfParams, sizeof(SColumn)); + dst->base.pParam = taosMemoryCalloc(src->base.numOfParams, sizeof(SColumn)); memcpy(dst->base.pParam, src->base.pParam, sizeof(SColumn) * src->base.numOfParams); // memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); @@ -5853,7 +5947,7 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { static SExprInfo* exprArrayDup(SArray* pExprList) { size_t numOfOutput = taosArrayGetSize(pExprList); - SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); + SExprInfo* p = taosMemoryCalloc(numOfOutput, sizeof(SExprInfo)); for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExpr = taosArrayGetP(pExprList, i); assignExprInfo(&p[i], pExpr); @@ -6087,11 +6181,11 @@ static SSDataBlock* doSortedMerge(SOperatorInfo *pOperator, bool* newgroup) { pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, p, pInfo->binfo.pRes->info.numOfCols, "GET_TASKID(pTaskInfo)"); - tfree(p); + taosMemoryFreeClear(p); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); for(int32_t i = 0; i < pOperator->numOfDownstream; ++i) { - SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource)); ps->param = pOperator->pDownstream[i]; tsortAddSource(pInfo->pSortHandle, ps); } @@ -6157,7 +6251,7 @@ static int32_t initGroupCol(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pGr ASSERT(taosArrayGetSize(pGroupInfo) == taosArrayGetSize(plist)); - pInfo->groupVal = calloc(1, (POINTER_BYTES * numOfGroupCol + len)); + pInfo->groupVal = taosMemoryCalloc(1, (POINTER_BYTES * numOfGroupCol + len)); if (pInfo->groupVal == NULL) { taosArrayDestroy(plist); return TSDB_CODE_OUT_OF_MEMORY; @@ -6177,8 +6271,8 @@ static int32_t initGroupCol(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pGr } SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo) { - SSortedMergeOperatorInfo* pInfo = calloc(1, sizeof(SSortedMergeOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SSortedMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortedMergeOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -6233,8 +6327,8 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t destroySortedMergeOperatorInfo(pInfo, num); } - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -6255,10 +6349,10 @@ static SSDataBlock* doSort(SOperatorInfo *pOperator, bool* newgroup) { pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage, p, pInfo->pDataBlock->info.numOfCols, "GET_TASKID(pTaskInfo)"); - tfree(p); + taosMemoryFreeClear(p); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); - SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource)); ps->param = pOperator; tsortAddSource(pInfo->pSortHandle, ps); @@ -6273,11 +6367,11 @@ static SSDataBlock* doSort(SOperatorInfo *pOperator, bool* newgroup) { } SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal, SExecTaskInfo* pTaskInfo) { - SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOrderOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SOrderOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -6296,9 +6390,9 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx } if (pInfo->orderInfo == NULL || pInfo->pDataBlock == NULL) { - tfree(pOperator); + taosMemoryFreeClear(pOperator); destroyOrderOperatorInfo(pInfo, numOfCols); - tfree(pInfo); + taosMemoryFreeClear(pInfo); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; @@ -6386,7 +6480,7 @@ static void aggEncodeResultRow(SOperatorInfo* pOperator, char **result, int32_t int32_t size = taosHashGetSize(pSup->pResultRowHashTable); size_t keyLen = POINTER_BYTES; // estimate the key length int32_t totalSize = sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); - *result = calloc(1, totalSize); + *result = taosMemoryCalloc(1, totalSize); if(*result == NULL){ terrno = TSDB_CODE_OUT_OF_MEMORY; return; @@ -6401,10 +6495,10 @@ static void aggEncodeResultRow(SOperatorInfo* pOperator, char **result, int32_t // recalculate the result size int32_t realTotalSize = offset + sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize; if (realTotalSize > totalSize){ - char *tmp = realloc(*result, realTotalSize); + char *tmp = taosMemoryRealloc(*result, realTotalSize); if (tmp == NULL){ terrno = TSDB_CODE_OUT_OF_MEMORY; - free(*result); + taosMemoryFree(*result); *result = NULL; return; }else{ @@ -6960,7 +7054,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI TSKEY* tsList = (TSKEY*)pTsColInfoData->pData; if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { pInfo->reptScan = true; - tfree(pInfo->prevData); + taosMemoryFreeClear(pInfo->prevData); } pInfo->numOfRows = 0; @@ -6970,7 +7064,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI continue; } if (pInfo->prevData == NULL) { - pInfo->prevData = malloc(bytes); + pInfo->prevData = taosMemoryMalloc(bytes); memcpy(pInfo->prevData, val, bytes); pInfo->numOfRows = 1; pInfo->curWindow.skey = tsList[j]; @@ -7304,19 +7398,19 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { destroyOperatorInfo(pOperator->pDownstream[i]); } - tfree(pOperator->pDownstream); + taosMemoryFreeClear(pOperator->pDownstream); pOperator->numOfDownstream = 0; } - tfree(pOperator->info); - tfree(pOperator); + taosMemoryFreeClear(pOperator->info); + taosMemoryFreeClear(pOperator); } int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t numOfOutput, const char* pKey) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pAggSup->keyBuf = calloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); + pAggSup->keyBuf = taosMemoryCalloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); @@ -7335,7 +7429,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx *pCtx, int32_t n } static void cleanupAggSup(SAggSupporter* pAggSup) { - tfree(pAggSup->keyBuf); + taosMemoryFreeClear(pAggSup->keyBuf); taosHashCleanup(pAggSup->pResultRowHashTable); taosHashCleanup(pAggSup->pResultRowListSet); taosArrayDestroy(pAggSup->pResultRowArrayList); @@ -7352,7 +7446,7 @@ static int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, S } static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInfo) { - STableQueryInfo* pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pTableQueryInfo == NULL) { return NULL; } @@ -7377,8 +7471,8 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7416,8 +7510,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* return pOperator; _error: destroyAggOperatorInfo(pInfo, numOfCols); - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -7426,7 +7520,7 @@ static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { assert(pInfo != NULL); destroySqlFunctionCtx(pInfo->pCtx, numOfOutput); - tfree(pInfo->rowCellInfoOffset); + taosMemoryFreeClear(pInfo->rowCellInfoOffset); cleanupResultRowInfo(&pInfo->resultRowInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); @@ -7440,7 +7534,7 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->prevData); + taosMemoryFreeClear(pInfo->prevData); } void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { @@ -7463,13 +7557,13 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); - tfree(pInfo->p); + taosMemoryFreeClear(pInfo->p); } void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - tfree(pInfo->keyBuf); + taosMemoryFreeClear(pInfo->keyBuf); taosArrayDestroy(pInfo->pGroupCols); taosArrayDestroy(pInfo->pGroupColVals); } @@ -7499,7 +7593,7 @@ static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*) param; taosHashCleanup(pInfo->pSet); - tfree(pInfo->buf); + taosMemoryFreeClear(pInfo->buf); taosArrayDestroy(pInfo->pDistinctDataInfo); pInfo->pRes = blockDataDestroy(pInfo->pRes); } @@ -7526,7 +7620,7 @@ void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { } SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); int32_t numOfRows = 1; int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, pTaskInfo->id.str); @@ -7538,7 +7632,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprI size_t tableGroup = taosArrayGetSize(pTableGroupInfo->pGroupList); initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableAggregate"; // pOperator->operatorType = OP_MultiTableAggregate; pOperator->blockingOptr = true; @@ -7562,8 +7656,8 @@ _error: } SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { - SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7603,7 +7697,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols) { #if 0 - SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo)); + SColumnInfo* pCols = taosMemoryCalloc(numOfOutput, sizeof(SColumnInfo)); int32_t numOfFilter = 0; for(int32_t i = 0; i < numOfOutput; ++i) { @@ -7617,7 +7711,7 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3 pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters; if (pCols[i].flist.numOfFilters != 0) { - pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); + pCols[i].flist.filterInfo = taosMemoryCalloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo)); } else { // avoid runtime error @@ -7635,8 +7729,8 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3 } SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo) { - SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SLimitOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SLimitOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7656,16 +7750,16 @@ SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit return pOperator; _error: - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7708,20 +7802,20 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* _error: destroyIntervalOperatorInfo(pInfo, numOfCols); - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; } SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); // pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllTimeIntervalAggOperator"; // pOperator->operatorType = OP_AllTimeWindow; @@ -7739,14 +7833,14 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S } SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); + SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); pInfo->colIndex = -1; pInfo->reptScan = false; pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); // pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "StateWindowOperator"; // pOperator->operatorType = OP_StateWindow; pOperator->blockingOptr = true; @@ -7763,8 +7857,8 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper } SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { - SSessionAggOperatorInfo* pInfo = calloc(1, sizeof(SSessionAggOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7798,20 +7892,20 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo destroySWindowOperatorInfo(pInfo, numOfCols); } - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; } SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); // pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableTimeIntervalOperator"; // pOperator->operatorType = OP_MultiTableTimeInterval; pOperator->blockingOptr = true; @@ -7829,13 +7923,13 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim } SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); // pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllMultiTableTimeIntervalOperator"; // pOperator->operatorType = OP_AllMultiTableTimeInterval; pOperator->blockingOptr = true; @@ -7868,7 +7962,7 @@ static int32_t initGroupOptrInfo(SGroupbyOperatorInfo *pInfo, SArray* pGroupColL key.bytes = pCol->bytes; key.type = pCol->type; key.isNull = false; - key.pData = calloc(1, pCol->bytes); + key.pData = taosMemoryCalloc(1, pCol->bytes); if (key.pData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -7877,7 +7971,7 @@ static int32_t initGroupOptrInfo(SGroupbyOperatorInfo *pInfo, SArray* pGroupColL } int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; - pInfo->keyBuf = calloc(1, pInfo->groupKeyLen + nullFlagSize); + pInfo->keyBuf = taosMemoryCalloc(1, pInfo->groupKeyLen + nullFlagSize); if (pInfo->keyBuf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -7888,8 +7982,8 @@ static int32_t initGroupOptrInfo(SGroupbyOperatorInfo *pInfo, SArray* pGroupColL SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { - SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -7918,8 +8012,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx return pOperator; _error: - tfree(pInfo); - tfree(pOperator); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); return NULL; } @@ -7939,7 +8033,7 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval->sliding, pInterval->slidingUnit, (int8_t)pInterval->precision, fillType, pColInfo, id); - pInfo->p = calloc(numOfCols, POINTER_BYTES); + pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -7950,8 +8044,8 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo) { - SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pInfo->pRes = pResBlock; pInfo->multigroupResult = multigroupResult; @@ -7980,14 +8074,14 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp return pOperator; _error: - tfree(pOperator); - tfree(pInfo); + taosMemoryFreeClear(pOperator); + taosMemoryFreeClear(pInfo); return NULL; } SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { - SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SSLimitOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSLimitOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); // pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); // pInfo->slimit = pQueryAttr->slimit; @@ -8005,7 +8099,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI } int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + pInfo->prevRow = taosMemoryCalloc(1, (POINTER_BYTES * numOfCols + len)); int32_t offset = POINTER_BYTES * numOfCols; for(int32_t i = 0; i < numOfCols; ++i) { @@ -8164,7 +8258,7 @@ static SSDataBlock* doTagScan(SOperatorInfo *pOperator, bool* newgroup) { } SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { - STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); + STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); // pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pResultInfo->capacity); size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); @@ -8173,7 +8267,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables; pInfo->curPos = 0; - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "SeqTableTagScan"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; pOperator->blockingOptr = false; @@ -8207,7 +8301,7 @@ static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* p } } pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); - pInfo->buf = calloc(1, pInfo->totalBytes); + pInfo->buf = taosMemoryCalloc(1, pInfo->totalBytes); return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; } @@ -8265,7 +8359,7 @@ static SSDataBlock* hashDistinct(SOperatorInfo *pOperator, bool* newgroup) { for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); - char* tmp = realloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); + char* tmp = taosMemoryRealloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); if (tmp == NULL) { return NULL; } else { @@ -8301,7 +8395,7 @@ static SSDataBlock* hashDistinct(SOperatorInfo *pOperator, bool* newgroup) { } SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { - SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); + SDistinctOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDistinctOperatorInfo)); pInfo->totalBytes = 0; pInfo->buf = NULL; pInfo->threshold = tsMaxNumOfDistinctResults; // distinct result threshold @@ -8311,7 +8405,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pOperator->name = "DistinctOperator"; pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; @@ -8376,7 +8470,7 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t if (pColFilter->filterstr) { pColFilter->len = htobe64(pFilterMsg->len); - pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + pColFilter->pz = (int64_t)taosMemoryCalloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator if (pColFilter->pz == 0) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -8415,7 +8509,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* } *numOfExprs = numOfFuncs + numOfGroupKeys; - SExprInfo* pExprs = calloc(*numOfExprs, sizeof(SExprInfo)); + SExprInfo* pExprs = taosMemoryCalloc(*numOfExprs, sizeof(SExprInfo)); for(int32_t i = 0; i < (*numOfExprs); ++i) { STargetNode* pTargetNode = NULL; @@ -8427,14 +8521,14 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* SExprInfo* pExp = &pExprs[pTargetNode->slotId]; - pExp->pExpr = calloc(1, sizeof(tExprNode)); + pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode)); pExp->pExpr->_function.num = 1; pExp->pExpr->_function.functionId = -1; - pExp->base.pParam = calloc(1, sizeof(SFunctParam)); + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); pExp->base.numOfParams = 1; - pExp->base.pParam[0].pCol = calloc(1, sizeof(SColumn)); + pExp->base.pParam[0].pCol = taosMemoryCalloc(1, sizeof(SColumn)); SColumn* pCol = pExp->base.pParam[0].pCol; // it is a project query, or group by column @@ -8478,13 +8572,13 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* } static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { - SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo)); + SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; - char* p = calloc(1, 128); + char* p = taosMemoryCalloc(1, 128); snprintf(p, 128, "TID:0x%"PRIx64" QID:0x%"PRIx64, taskId, queryId); pTaskInfo->id.str = strdup(p); @@ -8497,6 +8591,7 @@ static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractScanColumnId(SNodeList* pNodeList); static SArray* extractColumnInfo(SNodeList* pNodeList); +static SArray* extractColMatchInfo(SNodeList* pNodeList); SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -8505,7 +8600,9 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa size_t numOfCols = LIST_LENGTH(pScanPhyNode->pScanCols); tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); - return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); + SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols); + + return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createOutputBuf_rv1(pExchange->node.pOutputDataBlockDesc); @@ -8530,7 +8627,8 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SArray* colList = extractScanColumnId(pScanNode->pScanCols); SOperatorInfo* pOperator = createSysTableScanOperatorInfo(pHandle->meta, pResBlock, &pScanNode->tableName, - pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, pTaskInfo); + pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, + colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); return pOperator; } else { ASSERT(0); @@ -8607,7 +8705,7 @@ static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STa cond.order = pTableScanNode->scan.order; cond.numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo)); if (cond.colList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; @@ -8644,9 +8742,14 @@ SArray* extractScanColumnId(SNodeList* pNodeList) { } for(int32_t i = 0; i < numOfCols; ++i) { - STargetNode* pNode = (STargetNode*) nodesListGetNode(pNodeList, i); - SColumnNode* pColNode = (SColumnNode*) pNode->pExpr; - taosArrayPush(pList, &pColNode->colId); + for (int32_t j = 0; j < numOfCols; ++j) { + STargetNode* pNode = (STargetNode*) nodesListGetNode(pNodeList, j); + if (pNode->slotId == i) { + SColumnNode* pColNode = (SColumnNode*) pNode->pExpr; + taosArrayPush(pList, &pColNode->colId); + break; + } + } } return pList; @@ -8678,6 +8781,28 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { return pList; } +SArray* extractColMatchInfo(SNodeList* pNodeList) { + size_t numOfCols = LIST_LENGTH(pNodeList); + SArray* pList = taosArrayInit(numOfCols, sizeof(SColMatchInfo)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for(int32_t i = 0; i < numOfCols; ++i) { + STargetNode* pNode = (STargetNode*) nodesListGetNode(pNodeList, i); + SColumnNode* pColNode = (SColumnNode*) pNode->pExpr; + + SColMatchInfo c = {0}; + c.colId = pColNode->colId; + c.targetSlotId = pNode->slotId; + + taosArrayPush(pList, &c); + } + + return pList; +} + int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { int32_t code = 0; if (tableType == TSDB_SUPER_TABLE) { @@ -8752,7 +8877,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead return code; _complete: - tfree(*pTaskInfo); + taosMemoryFreeClear(*pTaskInfo); terrno = code; return code; @@ -8763,7 +8888,7 @@ int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int return TSDB_CODE_SUCCESS; } - *dst = calloc(filterNum, sizeof(*src)); + *dst = taosMemoryCalloc(filterNum, sizeof(*src)); if (*dst == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -8772,11 +8897,11 @@ int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int for (int32_t i = 0; i < filterNum; i++) { if ((*dst)[i].filterstr && dst[i]->len > 0) { - void *pz = calloc(1, (size_t)(*dst)[i].len + 1); + void *pz = taosMemoryCalloc(1, (size_t)(*dst)[i].len + 1); if (pz == NULL) { if (i == 0) { - free(*dst); + taosMemoryFree(*dst); } else { freeColumnFilterInfo(*dst, i); } @@ -8836,7 +8961,7 @@ int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { } //int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { -// *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); +// *pFilterInfo = taosMemoryCalloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); // if (*pFilterInfo == NULL) { // return TSDB_CODE_QRY_OUT_OF_MEMORY; // } @@ -8849,7 +8974,7 @@ int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { // pFilter->info = pCols[i]; // // pFilter->numOfFilters = pCols[i].flist.numOfFilters; -// pFilter->pFilters = calloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); +// pFilter->pFilters = taosMemoryCalloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); // if (pFilter->pFilters == NULL) { // return TSDB_CODE_QRY_OUT_OF_MEMORY; // } @@ -8891,11 +9016,11 @@ void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFil // if (pFilterInfo[i].pFilters->filterInfo.lowerRelOptr == TSDB_RELATION_IN) { // taosHashCleanup((SHashObj *)(pFilterInfo[i].pFilters->q)); // } -// tfree(pFilterInfo[i].pFilters); +// taosMemoryFreeClear(pFilterInfo[i].pFilters); // } // } // -// tfree(pFilterInfo); +// taosMemoryFreeClear(pFilterInfo); return NULL; } @@ -8987,11 +9112,11 @@ void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { for (int32_t i = 0; i < numOfFilters; i++) { if (pFilter[i].filterstr && pFilter[i].pz) { - free((void*)(pFilter[i].pz)); + taosMemoryFree((void*)(pFilter[i].pz)); } } - free(pFilter); + taosMemoryFree(pFilter); } static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) { @@ -9025,9 +9150,9 @@ void doDestroyTask(SExecTaskInfo *pTaskInfo) { // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); // taosHashCleanup(pTaskInfo->summary.operatorProfResults); - tfree(pTaskInfo->sql); - tfree(pTaskInfo->id.str); - tfree(pTaskInfo); + taosMemoryFreeClear(pTaskInfo->sql); + taosMemoryFreeClear(pTaskInfo->id.str); + taosMemoryFreeClear(pTaskInfo); } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 28319469cc..1b55467c85 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -209,7 +209,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { newLen += 4; } - char* p = realloc(pHashObj->pBucket, POINTER_BYTES * newLen); + char* p = taosMemoryRealloc(pHashObj->pBucket, POINTER_BYTES * newLen); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -219,7 +219,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { pHashObj->numOfAlloc = newLen; } - SLHashBucket* pBucket = calloc(1, sizeof(SLHashBucket)); + SLHashBucket* pBucket = taosMemoryCalloc(1, sizeof(SLHashBucket)); pHashObj->pBucket[pHashObj->numOfBuckets] = pBucket; pBucket->pPageIdList = taosArrayInit(2, sizeof(int32_t)); @@ -241,7 +241,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { } SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_t numOfTuplePerPage) { - SLHashObj* pHashObj = calloc(1, sizeof(SLHashObj)); + SLHashObj* pHashObj = taosMemoryCalloc(1, sizeof(SLHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -265,12 +265,12 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ pHashObj->tuplesPerPage = numOfTuplePerPage; pHashObj->numOfAlloc = 4; // initial allocated array list - pHashObj->pBucket = calloc(pHashObj->numOfAlloc, POINTER_BYTES); + pHashObj->pBucket = taosMemoryCalloc(pHashObj->numOfAlloc, POINTER_BYTES); code = doAddNewBucket(pHashObj); if (code != TSDB_CODE_SUCCESS) { destroyDiskbasedBuf(pHashObj->pBuf); - tfree(pHashObj); + taosMemoryFreeClear(pHashObj); terrno = code; return NULL; } @@ -282,11 +282,11 @@ void* tHashCleanup(SLHashObj* pHashObj) { destroyDiskbasedBuf(pHashObj->pBuf); for(int32_t i = 0; i < pHashObj->numOfBuckets; ++i) { taosArrayDestroy(pHashObj->pBucket[i]->pPageIdList); - tfree(pHashObj->pBucket[i]); + taosMemoryFreeClear(pHashObj->pBucket[i]); } - tfree(pHashObj->pBucket); - tfree(pHashObj); + taosMemoryFreeClear(pHashObj->pBucket); + taosMemoryFreeClear(pHashObj); return NULL; } diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 65067f58e3..981da0415e 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -29,7 +29,7 @@ #define FREE_HASH_NODE(_n) \ do { \ - tfree(_n); \ + taosMemoryFreeClear(_n); \ } while (0); typedef struct SHNode { @@ -62,7 +62,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t capacity = 4; } - SSHashObj* pHashObj = (SSHashObj*) calloc(1, sizeof(SSHashObj)); + SSHashObj* pHashObj = (SSHashObj*) taosMemoryCalloc(1, sizeof(SSHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -78,9 +78,9 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t pHashObj->keyLen = keyLen; pHashObj->dataLen = dataLen; - pHashObj->hashList = (SHNode **)calloc(pHashObj->capacity, sizeof(void *)); + pHashObj->hashList = (SHNode **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { - free(pHashObj); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -95,7 +95,7 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { } static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHNode *pNewNode = malloc(sizeof(SHNode) + keyLen + dsize); + SHNode *pNewNode = taosMemoryMalloc(sizeof(SHNode) + keyLen + dsize); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -120,7 +120,7 @@ void taosHashTableResize(SSHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { // qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; @@ -287,7 +287,7 @@ void tSimpleHashCleanup(SSHashObj *pHashObj) { } tSimpleHashClear(pHashObj); - tfree(pHashObj->hashList); + taosMemoryFreeClear(pHashObj->hashList); } size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index d424599758..85ba462c9a 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -65,7 +65,7 @@ typedef struct SSortHandle { static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param); static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { - SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.numOfCols = numOfCols; @@ -91,7 +91,7 @@ static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { * @return */ SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr) { - SSortHandle* pSortHandle = calloc(1, sizeof(SSortHandle)); + SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle)); pSortHandle->type = type; pSortHandle->pageSize = pageSize; @@ -118,8 +118,8 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { } destroyDiskbasedBuf(pSortHandle->pBuf); - tfree(pSortHandle->idStr); - tfree(pSortHandle); + taosMemoryFreeClear(pSortHandle->idStr); + taosMemoryFreeClear(pSortHandle); } int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { @@ -127,7 +127,7 @@ int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { } static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSources, SSDataBlock* pBlock, int32_t* sourceId) { - SExternalMemSource* pSource = calloc(1, sizeof(SExternalMemSource)); + SExternalMemSource* pSource = taosMemoryCalloc(1, sizeof(SExternalMemSource)); if (pSource == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -233,7 +233,7 @@ static int32_t sortComparClearup(SMsortComparParam* cmpParam) { for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { SExternalMemSource* pSource = cmpParam->pSources[i]; blockDataDestroy(pSource->src.pBlock); - tfree(pSource); + taosMemoryFreeClear(pSource); } cmpParam->numOfSources = 0; @@ -570,7 +570,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { } } - tfree(source); + taosMemoryFreeClear(source); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 2f1caab30b..7fb9b2ad7e 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -62,7 +62,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -70,8 +70,8 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { colInfo.info.type = TSDB_DATA_TYPE_INT; colInfo.info.bytes = sizeof(int32_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); @@ -82,7 +82,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { // // colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t); // colInfo1.varmeta.length = 0; -// colInfo1.varmeta.offset = static_cast(calloc(1, numOfRows * sizeof(int32_t))); +// colInfo1.varmeta.offset = static_cast(taosMemoryCalloc(1, numOfRows * sizeof(int32_t))); // // taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); } else { @@ -128,7 +128,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); @@ -136,8 +136,8 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; colInfo.info.bytes = sizeof(int64_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); -// colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); +// colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); @@ -146,8 +146,8 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { colInfo1.info.bytes = 4; colInfo1.info.colId = 2; - colInfo1.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo1.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + colInfo1.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo1.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); } else { @@ -195,7 +195,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_t rowsPerPage, int32_t type, int32_t numOfCols) { - SOperatorInfo* pOperator = static_cast(calloc(1, sizeof(SOperatorInfo))); + SOperatorInfo* pOperator = static_cast(taosMemoryCalloc(1, sizeof(SOperatorInfo))); pOperator->name = "dummyInputOpertor4Test"; if (numOfCols == 1) { @@ -204,7 +204,7 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_ pOperator->getNextFn = get2ColsDummyBlock; } - SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); + SDummyInputInfo *pInfo = (SDummyInputInfo*) taosMemoryCalloc(1, sizeof(SDummyInputInfo)); pInfo->totalPages = numOfBlocks; pInfo->startVal = startVal; pInfo->numOfRowsPerPage = rowsPerPage; @@ -958,11 +958,11 @@ TEST(testCase, inMem_sort_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); taosArrayPush(pExprInfo, &exp1); @@ -1002,10 +1002,10 @@ int32_t cmp(const void* p1, const void* p2) { #if 0 TEST(testCase, external_sort_Test) { #if 0 - su* v = static_cast(calloc(1000000, sizeof(su))); + su* v = static_cast(taosMemoryCalloc(1000000, sizeof(su))); for(int32_t i = 0; i < 1000000; ++i) { v[i].v = taosRand(); - v[i].c = static_cast(malloc(4)); + v[i].c = static_cast(taosMemoryMalloc(4)); *(int32_t*) v[i].c = i; } @@ -1027,11 +1027,11 @@ TEST(testCase, external_sort_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); // taosArrayPush(pExprInfo, &exp1); @@ -1071,8 +1071,8 @@ TEST(testCase, external_sort_Test) { printf("total:%ld\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); - tfree(exp); - tfree(exp1); + taosMemoryFreeClear(exp); + taosMemoryFreeClear(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } @@ -1088,21 +1088,21 @@ TEST(testCase, sorted_merge_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 1, "count_result"); - exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns = static_cast(taosMemoryCalloc(1, sizeof(SColumn))); exp->base.pColumns->flag = TSDB_COL_NORMAL; exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; exp->base.numOfCols = 1; taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); // taosArrayPush(pExprInfo, &exp1); int32_t numOfSources = 10; - SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*)); + SOperatorInfo** plist = (SOperatorInfo**) taosMemoryCalloc(numOfSources, sizeof(void*)); for(int32_t i = 0; i < numOfSources; ++i) { plist[i] = createDummyOperator(1, 1, 1, data_asc, 1); } @@ -1143,8 +1143,8 @@ TEST(testCase, sorted_merge_Test) { printf("total:%ld\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); - tfree(exp); - tfree(exp1); + taosMemoryFreeClear(exp); + taosMemoryFreeClear(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } @@ -1160,18 +1160,18 @@ TEST(testCase, time_interval_Operator_Test) { taosArrayPush(pOrderVal, &o); SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); - SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp->base.resSchema = createSchema(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, "ts"); - exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns = static_cast(taosMemoryCalloc(1, sizeof(SColumn))); exp->base.pColumns->flag = TSDB_COL_NORMAL; exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8}; exp->base.numOfCols = 1; taosArrayPush(pExprInfo, &exp); - SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + SExprInfo *exp1 = static_cast(taosMemoryCalloc(1, sizeof(SExprInfo))); exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, 8, 2, "res1"); - exp1->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp1->base.pColumns = static_cast(taosMemoryCalloc(1, sizeof(SColumn))); exp1->base.pColumns->flag = TSDB_COL_NORMAL; exp1->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; exp1->base.numOfCols = 1; @@ -1221,8 +1221,8 @@ TEST(testCase, time_interval_Operator_Test) { printf("total:%ld\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); - tfree(exp); - tfree(exp1); + taosMemoryFreeClear(exp); + taosMemoryFreeClear(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 1956794395..586aed7a67 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -47,15 +47,15 @@ SSDataBlock* getSingleColDummyBlock(void* param) { return NULL; } - SSDataBlock* pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); SColumnInfoData colInfo = {0}; colInfo.info.type = TSDB_DATA_TYPE_INT; colInfo.info.bytes = sizeof(int32_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(pInfo->pageRows, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(calloc(1, (pInfo->pageRows + 7) / 8)); + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8)); taosArrayPush(pBlock->pDataBlock, &colInfo); @@ -203,12 +203,12 @@ TEST(testCase, external_mem_sort_Test) { SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc"); tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); - _info* pInfo = (_info*) calloc(1, sizeof(_info)); + _info* pInfo = (_info*) taosMemoryCalloc(1, sizeof(_info)); pInfo->startVal = 100000; pInfo->pageRows = 1000; pInfo->count = 50; - SGenericSource* ps = static_cast(calloc(1, sizeof(SGenericSource))); + SGenericSource* ps = static_cast(taosMemoryCalloc(1, sizeof(SGenericSource))); ps->param = pInfo; tsortAddSource(phandle, ps); @@ -249,8 +249,8 @@ TEST(testCase, ordered_merge_sort_Test) { tsortSetComparFp(phandle, docomp); for(int32_t i = 0; i < 10; ++i) { - SGenericSource* p = static_cast(calloc(1, sizeof(SGenericSource))); - _info* c = static_cast<_info*>(calloc(1, sizeof(_info))); + SGenericSource* p = static_cast(taosMemoryCalloc(1, sizeof(SGenericSource))); + _info* c = static_cast<_info*>(taosMemoryCalloc(1, sizeof(_info))); c->count = 1; c->pageRows = 1000; c->startVal = 0; diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 6305d242ac..e1c7626f01 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -11,8 +11,8 @@ target_include_directories( target_link_libraries( function + PRIVATE os util common nodes scalar PUBLIC uv_a - PRIVATE os util common nodes ) add_executable(runUdf test/runUdf.c) @@ -21,6 +21,7 @@ target_include_directories( PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" "${CMAKE_SOURCE_DIR}/contrib/libuv/include" + "${CMAKE_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( @@ -34,6 +35,7 @@ target_include_directories( udf1 PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function" + "${CMAKE_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e27d53c1ee..70b2a48da7 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -15,6 +15,7 @@ #include "builtins.h" #include "builtinsimpl.h" +#include "scalar.h" #include "taoserror.h" #include "tdatablock.h" @@ -151,6 +152,136 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = lastFunction, .finalizeFunc = functionFinalize }, + { + .name = "abs", + .type = FUNCTION_TYPE_ABS, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = absFunction, + .finalizeFunc = NULL + }, + { + .name = "log", + .type = FUNCTION_TYPE_LOG, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = logFunction, + .finalizeFunc = NULL + }, + { + .name = "power", + .type = FUNCTION_TYPE_POW, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = powFunction, + .finalizeFunc = NULL + }, + { + .name = "sqrt", + .type = FUNCTION_TYPE_SQRT, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = sqrtFunction, + .finalizeFunc = NULL + }, + { + .name = "ceil", + .type = FUNCTION_TYPE_CEIL, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = ceilFunction, + .finalizeFunc = NULL + }, + { + .name = "floor", + .type = FUNCTION_TYPE_FLOOR, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = floorFunction, + .finalizeFunc = NULL + }, + { + .name = "round", + .type = FUNCTION_TYPE_ROUND, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = roundFunction, + .finalizeFunc = NULL + }, + { + .name = "sin", + .type = FUNCTION_TYPE_SIN, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = sinFunction, + .finalizeFunc = NULL + }, + { + .name = "cos", + .type = FUNCTION_TYPE_COS, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = cosFunction, + .finalizeFunc = NULL + }, + { + .name = "tan", + .type = FUNCTION_TYPE_TAN, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = tanFunction, + .finalizeFunc = NULL + }, + { + .name = "asin", + .type = FUNCTION_TYPE_ASIN, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = asinFunction, + .finalizeFunc = NULL + }, + { + .name = "acos", + .type = FUNCTION_TYPE_ACOS, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = acosFunction, + .finalizeFunc = NULL + }, + { + .name = "atan", + .type = FUNCTION_TYPE_ATAN, + .classification = FUNC_MGT_SCALAR_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = atanFunction, + .finalizeFunc = NULL + }, { .name = "concat", .type = FUNCTION_TYPE_CONCAT, diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 4360515328..05ed30c61a 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -1929,7 +1929,7 @@ static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { // set the corresponding tag data for each record // todo check malloc failure -// char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); +// char **pData = taosMemoryCalloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); // for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { // pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; // } @@ -1943,7 +1943,7 @@ static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { // } // } -// tfree(pData); +// taosMemoryFreeClear(pData); } /* @@ -2422,7 +2422,7 @@ static void apercentile_finalizer(SqlFunctionCtx *pCtx) { // double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); // // memcpy(pCtx->pOutput, res, sizeof(double)); -// free(res); +// taosMemoryFree(res); // } else { // setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); // return; @@ -2433,7 +2433,7 @@ static void apercentile_finalizer(SqlFunctionCtx *pCtx) { double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); memcpy(pCtx->pOutput, res, sizeof(double)); - free(res); + taosMemoryFree(res); } else { // no need to free setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; @@ -4004,7 +4004,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi char* outputBuf = NULL; if (comp) { - outputBuf = malloc(originalLen); + outputBuf = taosMemoryMalloc(originalLen); size_t actualLen = compLen; const char* compStr = tbufReadBinary(&br, &actualLen); @@ -4018,7 +4018,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); if (comp) { - tfree(outputBuf); + taosMemoryFreeClear(outputBuf); } } diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index bede8b80fd..814aa48b55 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -52,10 +52,10 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { } else if (pNode->nodeType == TEXPR_VALUE_NODE) { taosVariantDestroy(pNode->pVal); } else if (pNode->nodeType == TEXPR_COL_NODE) { - tfree(pNode->pSchema); + taosMemoryFreeClear(pNode->pSchema); } - free(pNode); + taosMemoryFree(pNode); } static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { @@ -80,12 +80,12 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { assert((*pExpr)->_node.pRight == NULL); } else if (type == TEXPR_VALUE_NODE) { taosVariantDestroy((*pExpr)->pVal); - free((*pExpr)->pVal); + taosMemoryFree((*pExpr)->pVal); } else if (type == TEXPR_COL_NODE) { - free((*pExpr)->pSchema); + taosMemoryFree((*pExpr)->pSchema); } - free(*pExpr); + taosMemoryFree(*pExpr); *pExpr = NULL; } @@ -154,7 +154,7 @@ void exprTreeToBinary(SBufferWriter* bw, tExprNode* expr) { // TODO: these three functions should be made global static void* exception_calloc(size_t nmemb, size_t size) { - void* p = calloc(nmemb, size); + void* p = taosMemoryCalloc(nmemb, size); if (p == NULL) { THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -162,7 +162,7 @@ static void* exception_calloc(size_t nmemb, size_t size) { } static void* exception_malloc(size_t size) { - void* p = malloc(size); + void* p = taosMemoryMalloc(size); if (p == NULL) { THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -195,7 +195,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) { pVal->nType = tbufReadUint32(br); if (pVal->nType == TSDB_DATA_TYPE_BINARY) { tbufReadToBuffer(br, &pVal->nLen, sizeof(pVal->nLen)); - pVal->pz = calloc(1, pVal->nLen + 1); + pVal->pz = taosMemoryCalloc(1, pVal->nLen + 1); tbufReadToBuffer(br, pVal->pz, pVal->nLen); } else { pVal->i = tbufReadInt64(br); @@ -377,7 +377,7 @@ void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t bufLen = 128; } - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + char *tmp = taosMemoryCalloc(1, bufLen * TSDB_NCHAR_SIZE); for (int32_t i = 0; i < sz; i++) { switch (sType) { @@ -440,7 +440,7 @@ void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t taosVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); if (bufLen < t) { - tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); + tmp = taosMemoryRealloc(tmp, t * TSDB_NCHAR_SIZE); bufLen = (int32_t)t; } @@ -530,7 +530,7 @@ void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t err_ret: taosVariantDestroy(&tmpVar); taosHashCleanup(pObj); - tfree(tmp); + taosMemoryFreeClear(tmp); } tExprNode* exprdup(tExprNode* pNode) { @@ -538,7 +538,7 @@ tExprNode* exprdup(tExprNode* pNode) { return NULL; } - tExprNode* pCloned = calloc(1, sizeof(tExprNode)); + tExprNode* pCloned = taosMemoryCalloc(1, sizeof(tExprNode)); if (pNode->nodeType == TEXPR_BINARYEXPR_NODE) { tExprNode* pLeft = exprdup(pNode->_node.pLeft); tExprNode* pRight = exprdup(pNode->_node.pRight); @@ -547,17 +547,17 @@ tExprNode* exprdup(tExprNode* pNode) { pCloned->_node.pRight = pRight; pCloned->_node.optr = pNode->_node.optr; } else if (pNode->nodeType == TEXPR_VALUE_NODE) { - pCloned->pVal = calloc(1, sizeof(SVariant)); + pCloned->pVal = taosMemoryCalloc(1, sizeof(SVariant)); taosVariantAssign(pCloned->pVal, pNode->pVal); } else if (pNode->nodeType == TEXPR_COL_NODE) { - pCloned->pSchema = calloc(1, sizeof(SSchema)); + pCloned->pSchema = taosMemoryCalloc(1, sizeof(SSchema)); *pCloned->pSchema = *pNode->pSchema; } else if (pNode->nodeType == TEXPR_FUNCTION_NODE) { strcpy(pCloned->_function.functionName, pNode->_function.functionName); int32_t num = pNode->_function.num; pCloned->_function.num = num; - pCloned->_function.pChild = calloc(num, POINTER_BYTES); + pCloned->_function.pChild = taosMemoryCalloc(num, POINTER_BYTES); for(int32_t i = 0; i < num; ++i) { pCloned->_function.pChild[i] = exprdup(pNode->_function.pChild[i]); } diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index 9b3dca7393..1f7e83286b 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -149,7 +149,7 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { return; } - *next = calloc(1, pFillInfo->rowSize); + *next = taosMemoryCalloc(1, pFillInfo->rowSize); for (int i = 1; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes); @@ -258,7 +258,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) { /* the raw data block is exhausted, next value does not exists */ if (pFillInfo->index >= pFillInfo->numOfRows) { - tfree(*next); + taosMemoryFreeClear(*next); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; @@ -314,7 +314,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t pSchema->type = pColInfo->col.type; pSchema->bytes = pColInfo->col.bytes; - pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes); + pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pColInfo->col.bytes); pColInfo->tagIndex = k; k += 1; @@ -347,7 +347,7 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag return NULL; } - SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo)); + SFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SFillInfo)); taosResetFillInfo(pFillInfo, skey); pFillInfo->order = order; @@ -364,10 +364,10 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag pFillInfo->interval.sliding = slidingTime; pFillInfo->interval.slidingUnit = slidingUnit; - pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); + pFillInfo->pData = taosMemoryMalloc(POINTER_BYTES * numOfCols); // if (numOfTags > 0) { - pFillInfo->pTags = calloc(numOfCols, sizeof(SFillTagColInfo)); + pFillInfo->pTags = taosMemoryCalloc(numOfCols, sizeof(SFillTagColInfo)); for (int32_t i = 0; i < numOfCols; ++i) { pFillInfo->pTags[i].col.colId = -2; // TODO } @@ -394,19 +394,19 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { return NULL; } - tfree(pFillInfo->prevValues); - tfree(pFillInfo->nextValues); + taosMemoryFreeClear(pFillInfo->prevValues); + taosMemoryFreeClear(pFillInfo->nextValues); for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { - tfree(pFillInfo->pTags[i].tagVal); + taosMemoryFreeClear(pFillInfo->pTags[i].tagVal); } - tfree(pFillInfo->pTags); + taosMemoryFreeClear(pFillInfo->pTags); - tfree(pFillInfo->pData); - tfree(pFillInfo->pFillCol); + taosMemoryFreeClear(pFillInfo->pData); + taosMemoryFreeClear(pFillInfo->pFillCol); - tfree(pFillInfo); + taosMemoryFreeClear(pFillInfo); return NULL; } @@ -530,7 +530,7 @@ int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal) { int32_t offset = 0; - struct SFillColInfo* pFillCol = calloc(numOfOutput, sizeof(SFillColInfo)); + struct SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); if (pFillCol == NULL) { return NULL; } diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index 49799aef7a..2e60498aba 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -34,11 +34,11 @@ static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { /* need one redundant slot */ - SHistogramInfo* pHisto = malloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); + SHistogramInfo* pHisto = taosMemoryMalloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); #if !defined(USE_ARRAYLIST) pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - SInsertSupporter* pss = malloc(sizeof(SInsertSupporter)); + SInsertSupporter* pss = taosMemoryMalloc(sizeof(SInsertSupporter)); pss->numOfEntries = pHisto->maxEntries; pss->pSkipList = pHisto->pList; @@ -96,7 +96,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } #else tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); - SHistBin* entry = calloc(1, sizeof(SHistBin)); + SHistBin* entry = taosMemoryCalloc(1, sizeof(SHistBin)); entry->val = val; tSkipListNode* pResNode = SSkipListPut((*pHisto)->pList, entry, &key, 0); @@ -352,7 +352,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto) { return; } - free(*pHisto); + taosMemoryFree(*pHisto); *pHisto = NULL; } @@ -417,7 +417,7 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { #if defined(USE_ARRAYLIST) - double* pVal = malloc(num * sizeof(double)); + double* pVal = taosMemoryMalloc(num * sizeof(double)); for (int32_t i = 0; i < num; ++i) { double numOfElem = (ratio[i] / 100) * pHisto->numOfElems; @@ -463,7 +463,7 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { } } #else - double* pVal = malloc(num * sizeof(double)); + double* pVal = taosMemoryMalloc(num * sizeof(double)); for (int32_t i = 0; i < num; ++i) { double numOfElem = ratio[i] * pHisto->numOfElems; @@ -535,7 +535,7 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 return pResHistogram; } - SHistBin* pHistoBins = calloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); + SHistBin* pHistoBins = taosMemoryCalloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); int32_t i = 0, j = 0, k = 0; while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) { @@ -573,6 +573,6 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 pResHistogram->numOfEntries = k; memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k); - free(pHistoBins); + taosMemoryFree(pHistoBins); return pResHistogram; } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 06c58430a4..dd57024624 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -29,7 +29,7 @@ int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { } static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) { - SFilePage *buffer = (SFilePage *)calloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); + SFilePage *buffer = (SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); @@ -216,7 +216,7 @@ static void resetSlotInfo(tMemBucket* pBucket) { } tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) { - tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket)); + tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket)); if (pBucket == NULL) { return NULL; } @@ -233,7 +233,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { // qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); - free(pBucket); + taosMemoryFree(pBucket); return NULL; } @@ -243,13 +243,13 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, pBucket->hashFunc = getHashFunc(pBucket->type); if (pBucket->hashFunc == NULL) { // qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); - free(pBucket); + taosMemoryFree(pBucket); return NULL; } - pBucket->pSlots = (tMemBucketSlot *)calloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); + pBucket->pSlots = (tMemBucketSlot *)taosMemoryCalloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); if (pBucket->pSlots == NULL) { - free(pBucket); + taosMemoryFree(pBucket); return NULL; } @@ -271,8 +271,8 @@ void tMemBucketDestroy(tMemBucket *pBucket) { } destroyDiskbasedBuf(pBucket->pBuffer); - tfree(pBucket->pSlots); - tfree(pBucket); + taosMemoryFreeClear(pBucket->pSlots); + taosMemoryFreeClear(pBucket); } void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { @@ -449,7 +449,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal); double val = (1 - fraction) * td + fraction * nd; - tfree(buffer); + taosMemoryFreeClear(buffer); return val; } else { // incur a second round bucket split diff --git a/source/libs/function/src/tscript.c b/source/libs/function/src/tscript.c index 25cf3cfbb1..93e8ecf5cf 100644 --- a/source/libs/function/src/tscript.c +++ b/source/libs/function/src/tscript.c @@ -195,7 +195,7 @@ void taosLoadScriptDestroy(void *pInit) { } ScriptCtx* createScriptCtx(char *script, int8_t resType, int16_t resBytes) { - ScriptCtx *pCtx = (ScriptCtx *)calloc(1, sizeof(ScriptCtx)); + ScriptCtx *pCtx = (ScriptCtx *)taosMemoryCalloc(1, sizeof(ScriptCtx)); pCtx->state = SCRIPT_STATE_INIT; pCtx->pEnv = getScriptEnvFromPool(); // pCtx->resType = resType; @@ -229,7 +229,7 @@ ScriptCtx* createScriptCtx(char *script, int8_t resType, int16_t resBytes) { void destroyScriptCtx(void *pCtx) { if (pCtx == NULL) return; addScriptEnvToPool(((ScriptCtx *)pCtx)->pEnv); - free(pCtx); + taosMemoryFree(pCtx); } void luaValueToTaosType(lua_State *lua, char *interBuf, int32_t *numOfOutput, int16_t oType, int16_t oBytes) { @@ -332,12 +332,12 @@ void destroyLuaEnv(lua_State *lua) { int32_t scriptEnvPoolInit() { const int size = 10; // configure or not - pool = malloc(sizeof(ScriptEnvPool)); + pool = taosMemoryMalloc(sizeof(ScriptEnvPool)); taosThreadMutexInit(&pool->mutex, NULL); pool->scriptEnvs = tdListNew(sizeof(ScriptEnv *)); for (int i = 0; i < size; i++) { - ScriptEnv *env = malloc(sizeof(ScriptEnv)); + ScriptEnv *env = taosMemoryMalloc(sizeof(ScriptEnv)); env->funcId = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);; env->lua_state = createLuaEnv(); tdListAppend(pool->scriptEnvs, (void *)(&env)); @@ -360,13 +360,13 @@ void scriptEnvPoolCleanup() { } tdListFree(pool->scriptEnvs); taosThreadMutexDestroy(&pool->mutex); - free(pool); + taosMemoryFree(pool); } void destroyScriptEnv(ScriptEnv *pEnv) { destroyLuaEnv(pEnv->lua_state); taosHashCleanup(pEnv->funcId); - free(pEnv); + taosMemoryFree(pEnv); } ScriptEnv* getScriptEnvFromPool() { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 6fcdb34529..a1030f6c21 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -195,7 +195,7 @@ int32_t encodeRequest(char **pBuf, int32_t *pBufLen, SUdfRequest *request) { break; } - char *bufBegin = malloc(len); + char *bufBegin = taosMemoryMalloc(len); char *buf = bufBegin; //skip msgLen first @@ -263,7 +263,7 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { return -1; } char *buf = bufMsg; - SUdfRequest *request = malloc(sizeof(SUdfRequest)); + SUdfRequest *request = taosMemoryMalloc(sizeof(SUdfRequest)); request->subReq = NULL; request->msgLen = *(int32_t *) (buf); buf += sizeof(int32_t); @@ -274,7 +274,7 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { switch (request->type) { case UDF_TASK_SETUP: { - SUdfSetupRequest *setup = malloc(sizeof(SUdfSetupRequest)); + SUdfSetupRequest *setup = taosMemoryMalloc(sizeof(SUdfSetupRequest)); memcpy(setup->udfName, buf, 16); buf += 16; @@ -291,7 +291,7 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { break; } case UDF_TASK_CALL: { - SUdfCallRequest *call = malloc(sizeof(SUdfCallRequest)); + SUdfCallRequest *call = taosMemoryMalloc(sizeof(SUdfCallRequest)); call->udfHandle = *(int64_t *) buf; buf += sizeof(int64_t); @@ -311,7 +311,7 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { } case UDF_TASK_TEARDOWN: { - SUdfTeardownRequest *teardown = malloc(sizeof(SUdfTeardownRequest)); + SUdfTeardownRequest *teardown = taosMemoryMalloc(sizeof(SUdfTeardownRequest)); teardown->udfHandle = *(int64_t *) buf; buf += sizeof(int64_t); @@ -322,8 +322,8 @@ int32_t decodeRequest(char *bufMsg, int32_t bufLen, SUdfRequest **pRequest) { } if (buf - bufMsg != bufLen) { debugPrint("%s", "decode request error"); - free(request->subReq); - free(request); + taosMemoryFree(request->subReq); + taosMemoryFree(request); return -1; } *pRequest = request; @@ -352,7 +352,7 @@ int32_t encodeResponse(char **pBuf, int32_t *pBufLen, SUdfResponse *response) { } } - char *bufBegin = malloc(len); + char *bufBegin = taosMemoryMalloc(len); char *buf = bufBegin; //skip msgLen @@ -408,7 +408,7 @@ int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { return -1; } char *buf = bufMsg; - SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); rsp->msgLen = *(int32_t *) buf; buf += sizeof(int32_t); rsp->seqNum = *(int64_t *) buf; @@ -420,14 +420,14 @@ int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { switch (rsp->type) { case UDF_TASK_SETUP: { - SUdfSetupResponse *setupRsp = (SUdfSetupResponse *) malloc(sizeof(SUdfSetupResponse)); + SUdfSetupResponse *setupRsp = (SUdfSetupResponse *) taosMemoryMalloc(sizeof(SUdfSetupResponse)); setupRsp->udfHandle = *(int64_t *) buf; buf += sizeof(int64_t); rsp->subRsp = (char *) setupRsp; break; } case UDF_TASK_CALL: { - SUdfCallResponse *callRsp = (SUdfCallResponse *) malloc(sizeof(SUdfCallResponse)); + SUdfCallResponse *callRsp = (SUdfCallResponse *) taosMemoryMalloc(sizeof(SUdfCallResponse)); callRsp->outputBytes = *(int32_t *) buf; buf += sizeof(int32_t); @@ -444,7 +444,7 @@ int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { break; } case UDF_TASK_TEARDOWN: { - SUdfTeardownResponse *teardownRsp = (SUdfTeardownResponse *) malloc(sizeof(SUdfTeardownResponse)); + SUdfTeardownResponse *teardownRsp = (SUdfTeardownResponse *) taosMemoryMalloc(sizeof(SUdfTeardownResponse)); rsp->subRsp = teardownRsp; break; } @@ -453,8 +453,8 @@ int32_t decodeResponse(char *bufMsg, int32_t bufLen, SUdfResponse **pResponse) { } if (buf - bufMsg != bufLen) { debugPrint("%s", "can not decode response"); - free(rsp->subRsp); - free(rsp); + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); return -1; } *pResponse = rsp; @@ -475,9 +475,9 @@ void onUdfcPipeClose(uv_handle_t *handle) { uv_sem_post(&task->taskSem); } - free(conn->readBuf.buf); - free(conn); - free((uv_pipe_t *) handle); + taosMemoryFree(conn->readBuf.buf); + taosMemoryFree(conn); + taosMemoryFree((uv_pipe_t *) handle); } @@ -511,9 +511,9 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT } // TODO: the call buffer is setup and freed by udf invocation - free(uvTask->rspBuf.base); - free(rsp->subRsp); - free(rsp); + taosMemoryFree(uvTask->rspBuf.base); + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); } else { task->errCode = uvTask->errCode; } @@ -532,7 +532,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); if (connBuf->cap == 0) { - connBuf->buf = malloc(msgHeadSize); + connBuf->buf = taosMemoryMalloc(msgHeadSize); if (connBuf->buf) { connBuf->len = 0; connBuf->cap = msgHeadSize; @@ -547,7 +547,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf } } else { connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap; - void *resultBuf = realloc(connBuf->buf, connBuf->cap); + void *resultBuf = taosMemoryRealloc(connBuf->buf, connBuf->cap); if (resultBuf) { connBuf->buf = resultBuf; buf->base = connBuf->buf + connBuf->len; @@ -648,8 +648,8 @@ void onUdfClientWrite(uv_write_t *write, int status) { //TODO Log error; } debugPrint("\tlength:%zu", uvTask->reqBuf.len); - free(write); - free(uvTask->reqBuf.base); + taosMemoryFree(write); + taosMemoryFree(uvTask->reqBuf.base); } void onUdfClientConnect(uv_connect_t *connect, int status) { @@ -659,12 +659,12 @@ void onUdfClientConnect(uv_connect_t *connect, int status) { //TODO: LOG error } uv_read_start((uv_stream_t *) uvTask->pipe, udfcAllocateBuffer, onUdfcRead); - free(connect); + taosMemoryFree(connect); uv_sem_post(&uvTask->taskSem); } int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { - SClientUvTaskNode *uvTask = calloc(1, sizeof(SClientUvTaskNode)); + SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode)); uvTask->type = uvTaskType; if (uvTaskType == UV_TASK_CONNECT) { @@ -718,11 +718,11 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { debugPrint("%s, type %d", "start uv task ", uvTask->type); switch (uvTask->type) { case UV_TASK_CONNECT: { - uv_pipe_t *pipe = malloc(sizeof(uv_pipe_t)); + uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); uv_pipe_init(&gUdfdLoop, pipe, 0); uvTask->pipe = pipe; - SClientUvConn *conn = malloc(sizeof(SClientUvConn)); + SClientUvConn *conn = taosMemoryMalloc(sizeof(SClientUvConn)); conn->pipe = pipe; conn->readBuf.len = 0; conn->readBuf.cap = 0; @@ -732,7 +732,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { pipe->data = conn; - uv_connect_t *connReq = malloc(sizeof(uv_connect_t)); + uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); connReq->data = uvTask; uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect); @@ -740,7 +740,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { } case UV_TASK_REQ_RSP: { uv_pipe_t *pipe = uvTask->pipe; - uv_write_t *write = malloc(sizeof(uv_write_t)); + uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); write->data = uvTask; uv_write(write, (uv_stream_t *) pipe, &uvTask->reqBuf, 1, onUdfClientWrite); break; @@ -833,16 +833,16 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { if (uvTaskType == UV_TASK_CONNECT) { task->session->udfSvcPipe = uvTask->pipe; } - free(uvTask); + taosMemoryFree(uvTask); uvTask = NULL; return task->errCode; } int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { debugPrint("%s", "client setup udf"); - SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); + SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; - task->session = malloc(sizeof(SUdfUvSession)); + task->session = taosMemoryMalloc(sizeof(SUdfUvSession)); task->type = UDF_TASK_SETUP; SUdfSetupRequest *req = &task->_setup.req; @@ -864,7 +864,7 @@ int32_t setupUdf(SUdfInfo *udfInfo, UdfHandle *handle) { task->session->severHandle = rsp->udfHandle; *handle = task->session; int32_t err = task->errCode; - free(task); + taosMemoryFree(task); return err; } @@ -872,7 +872,7 @@ int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, S int32_t *newStateSize, SUdfDataBlock *output) { debugPrint("%s", "client call udf"); - SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); + SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; task->session = (SUdfUvSession *) handle; task->type = UDF_TASK_CALL; @@ -894,14 +894,14 @@ int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, S output->size = rsp->outputBytes; output->data = rsp->output; int32_t err = task->errCode; - free(task); + taosMemoryFree(task); return err; } int32_t teardownUdf(UdfHandle handle) { debugPrint("%s", "client teardown udf"); - SClientUdfTask *task = malloc(sizeof(SClientUdfTask)); + SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; task->session = (SUdfUvSession *) handle; task->type = UDF_TASK_TEARDOWN; @@ -918,8 +918,8 @@ int32_t teardownUdf(UdfHandle handle) { udfcRunUvTask(task, UV_TASK_DISCONNECT); - free(task->session); - free(task); + taosMemoryFree(task->session); + taosMemoryFree(task); return err; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 27385325f5..b473f060c0 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -62,7 +62,7 @@ void udfdProcessRequest(uv_work_t *req) { switch (request->type) { case UDF_TASK_SETUP: { debugPrint("%s", "process setup request"); - SUdf *udf = malloc(sizeof(SUdf)); + SUdf *udf = taosMemoryMalloc(sizeof(SUdf)); udf->refCount = 0; SUdfSetupRequest *setup = request->subReq; strcpy(udf->name, setup->udfName); @@ -78,15 +78,15 @@ void udfdProcessRequest(uv_work_t *req) { //TODO find all functions normal, init, destroy, normal, merge, finalize uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->normalFunc)); - SUdfHandle *handle = malloc(sizeof(SUdfHandle)); + SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle)); handle->udf = udf; udf->refCount++; //TODO: allocate private structure and call init function and set it to handle - SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); rsp->seqNum = request->seqNum; rsp->type = request->type; rsp->code = 0; - SUdfSetupResponse *subRsp = malloc(sizeof(SUdfSetupResponse)); + SUdfSetupResponse *subRsp = taosMemoryMalloc(sizeof(SUdfSetupResponse)); subRsp->udfHandle = (int64_t) (handle); rsp->subRsp = subRsp; char *buf; @@ -95,11 +95,11 @@ void udfdProcessRequest(uv_work_t *req) { uvUdf->output = uv_buf_init(buf, len); - free(rsp->subRsp); - free(rsp); - free(request->subReq); - free(request); - free(uvUdf->input.base); + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + taosMemoryFree(uvUdf->input.base); break; } @@ -115,11 +115,11 @@ void udfdProcessRequest(uv_work_t *req) { //TODO: call different functions according to the step udf->normalFunc(call->step, call->state, call->stateBytes, input, &newState, &newStateSize, &output); - SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); rsp->seqNum = request->seqNum; rsp->type = request->type; rsp->code = 0; - SUdfCallResponse *subRsp = malloc(sizeof(SUdfCallResponse)); + SUdfCallResponse *subRsp = taosMemoryMalloc(sizeof(SUdfCallResponse)); subRsp->outputBytes = output.size; subRsp->output = output.data; subRsp->newStateBytes = newStateSize; @@ -131,13 +131,13 @@ void udfdProcessRequest(uv_work_t *req) { encodeResponse(&buf, &len, rsp); uvUdf->output = uv_buf_init(buf, len); - free(rsp->subRsp); - free(rsp); - free(newState); - free(output.data); - free(request->subReq); - free(request); - free(uvUdf->input.base); + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + taosMemoryFree(newState); + taosMemoryFree(output.data); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + taosMemoryFree(uvUdf->input.base); break; } case UDF_TASK_TEARDOWN: { @@ -150,26 +150,26 @@ void udfdProcessRequest(uv_work_t *req) { if (udf->refCount == 0) { uv_dlclose(&udf->lib); } - free(udf); + taosMemoryFree(udf); //TODO: call destroy and free udf private - free(handle); + taosMemoryFree(handle); - SUdfResponse *rsp = malloc(sizeof(SUdfResponse)); + SUdfResponse *rsp = taosMemoryMalloc(sizeof(SUdfResponse)); rsp->seqNum = request->seqNum; rsp->type = request->type; rsp->code = 0; - SUdfTeardownResponse *subRsp = malloc(sizeof(SUdfTeardownResponse)); + SUdfTeardownResponse *subRsp = taosMemoryMalloc(sizeof(SUdfTeardownResponse)); rsp->subRsp = subRsp; char *buf; int32_t len; encodeResponse(&buf, &len, rsp); uvUdf->output = uv_buf_init(buf, len); - free(rsp->subRsp); - free(rsp); - free(request->subReq); - free(request); - free(uvUdf->input.base); + taosMemoryFree(rsp->subRsp); + taosMemoryFree(rsp); + taosMemoryFree(request->subReq); + taosMemoryFree(request); + taosMemoryFree(uvUdf->input.base); break; } default: { @@ -187,9 +187,9 @@ void udfdOnWrite(uv_write_t *req, int status) { } SUvUdfWork *work = (SUvUdfWork *) req->data; debugPrint("\tlength: %zu", work->output.len); - free(work->output.base); - free(work); - free(req); + taosMemoryFree(work->output.base); + taosMemoryFree(work); + taosMemoryFree(req); } @@ -197,11 +197,11 @@ void udfdSendResponse(uv_work_t *work, int status) { debugPrint("%s", "send response"); SUvUdfWork *udfWork = (SUvUdfWork *) (work->data); - uv_write_t *write_req = malloc(sizeof(uv_write_t)); + uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); write_req->data = udfWork; uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); - free(work); + taosMemoryFree(work); } void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { @@ -209,7 +209,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { SUdfdUvConn *ctx = handle->data; int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t); if (ctx->inputCap == 0) { - ctx->inputBuf = malloc(msgHeadSize); + ctx->inputBuf = taosMemoryMalloc(msgHeadSize); if (ctx->inputBuf) { ctx->inputLen = 0; ctx->inputCap = msgHeadSize; @@ -224,7 +224,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { } } else { ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; - void *inputBuf = realloc(ctx->inputBuf, ctx->inputCap); + void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); if (inputBuf) { ctx->inputBuf = inputBuf; buf->base = ctx->inputBuf + ctx->inputLen; @@ -250,8 +250,8 @@ bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) { } void udfdHandleRequest(SUdfdUvConn *conn) { - uv_work_t *work = malloc(sizeof(uv_work_t)); - SUvUdfWork *udfWork = malloc(sizeof(SUvUdfWork)); + uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t)); + SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); udfWork->client = conn->client; udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); conn->inputBuf = NULL; @@ -264,9 +264,9 @@ void udfdHandleRequest(SUdfdUvConn *conn) { void udfdPipeCloseCb(uv_handle_t *pipe) { SUdfdUvConn *conn = pipe->data; - free(conn->client); - free(conn->inputBuf); - free(conn); + taosMemoryFree(conn->client); + taosMemoryFree(conn->inputBuf); + taosMemoryFree(conn); } void udfdUvHandleError(SUdfdUvConn *conn) { @@ -307,10 +307,10 @@ void udfdOnNewConnection(uv_stream_t *server, int status) { return; } - uv_pipe_t *client = (uv_pipe_t *) malloc(sizeof(uv_pipe_t)); + uv_pipe_t *client = (uv_pipe_t *) taosMemoryMalloc(sizeof(uv_pipe_t)); uv_pipe_init(loop, client, 0); if (uv_accept(server, (uv_stream_t *) client) == 0) { - SUdfdUvConn *ctx = malloc(sizeof(SUdfdUvConn)); + SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); ctx->client = (uv_stream_t *) client; ctx->inputBuf = 0; ctx->inputLen = 0; diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index b7d651e55e..bd742d23d0 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -3,6 +3,7 @@ #include #include "uv.h" +#include "os.h" #include "tudf.h" int main(int argc, char *argv[]) { @@ -28,8 +29,8 @@ int main(int argc, char *argv[]) { int callCount = 2; if (argc > 1) dataSize = atoi(argv[1]); if (argc > 2) callCount = atoi(argv[2]); - char *state = malloc(dataSize); - char *input = malloc(dataSize); + char *state = taosMemoryMalloc(dataSize); + char *input = taosMemoryMalloc(dataSize); SUdfDataBlock blockInput = {.data = input, .size = dataSize}; SUdfDataBlock blockOutput; char* newState; @@ -37,8 +38,8 @@ int main(int argc, char *argv[]) { for (int l = 0; l < callCount; ++l) { callUdf(handle, 0, state, dataSize, blockInput, &newState, &newStateSize, &blockOutput); } - free(state); - free(input); + taosMemoryFree(state); + taosMemoryFree(input); teardownUdf(handle); stopUdfService(); diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index 15e96e3bd1..dc88e8cf3e 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -2,17 +2,18 @@ #include #include +#include "os.h" #include "tudf.h" void udf1(int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newState, int32_t *newStateSize, SUdfDataBlock *output) { fprintf(stdout, "%s, step:%d\n", "udf function called", step); - char *newStateBuf = malloc(stateSize); + char *newStateBuf = taosMemoryMalloc(stateSize); memcpy(newStateBuf, state, stateSize); *newState = newStateBuf; *newStateSize = stateSize; - char *outputBuf = malloc(input.size); + char *outputBuf = taosMemoryMalloc(input.size); memcpy(outputBuf, input.data, input.size); output->data = outputBuf; output->size = input.size; diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 83410306eb..61dd952381 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -62,7 +62,7 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, Iterat int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { taosThreadOnce(&isInit, indexInit); - SIndex* sIdx = calloc(1, sizeof(SIndex)); + SIndex* sIdx = taosMemoryCalloc(1, sizeof(SIndex)); if (sIdx == NULL) { return -1; } @@ -115,8 +115,8 @@ void indexClose(SIndex* sIdx) { taosThreadMutexDestroy(&sIdx->mtx); indexTFileDestroy(sIdx->tindex); #endif - free(sIdx->path); - free(sIdx); + taosMemoryFree(sIdx->path); + taosMemoryFree(sIdx); return; } @@ -179,16 +179,16 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result EIndexOperatorType opera = multiQuerys->opera; int nQuery = taosArrayGetSize(multiQuerys->query); - char** fields = malloc(sizeof(char*) * nQuery); - char** keys = malloc(sizeof(char*) * nQuery); - int* types = malloc(sizeof(int) * nQuery); + char** fields = taosMemoryMalloc(sizeof(char*) * nQuery); + char** keys = taosMemoryMalloc(sizeof(char*) * nQuery); + int* types = taosMemoryMalloc(sizeof(int) * nQuery); for (int i = 0; i < nQuery; i++) { SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i); SIndexTerm* term = p->field_value; - fields[i] = calloc(1, term->nKey + 1); - keys[i] = calloc(1, term->nVal + 1); + fields[i] = taosMemoryCalloc(1, term->nKey + 1); + keys[i] = taosMemoryCalloc(1, term->nVal + 1); memcpy(fields[i], term->key, term->nKey); memcpy(keys[i], term->val, term->nVal); @@ -203,12 +203,12 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result } for (int i = 0; i < nQuery; i++) { - free(fields[i]); - free(keys[i]); + taosMemoryFree(fields[i]); + taosMemoryFree(keys[i]); } - free(fields); - free(keys); - free(types); + taosMemoryFree(fields); + taosMemoryFree(keys); + taosMemoryFree(types); #endif #ifdef USE_INVERTED_INDEX @@ -258,7 +258,7 @@ void indexOptsDestroy(SIndexOpts* opts) { * */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { - SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); + SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery)); if (p == NULL) { return NULL; } @@ -272,7 +272,7 @@ void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { indexTermDestroy(p->term); } taosArrayDestroy(pQuery->query); - free(pQuery); + taosMemoryFree(pQuery); }; int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) { SIndexTermQuery q = {.qType = qType, .term = term}; @@ -282,7 +282,7 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { - SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); + SIndexTerm* t = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); if (t == NULL) { return NULL; } @@ -291,19 +291,19 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy t->operType = oper; t->colType = colType; - t->colName = (char*)calloc(1, nColName + 1); + t->colName = (char*)taosMemoryCalloc(1, nColName + 1); memcpy(t->colName, colName, nColName); t->nColName = nColName; - t->colVal = (char*)calloc(1, nColVal + 1); + t->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); memcpy(t->colVal, colVal, nColVal); t->nColVal = nColVal; return t; } void indexTermDestroy(SIndexTerm* p) { - free(p->colName); - free(p->colVal); - free(p); + taosMemoryFree(p->colName); + taosMemoryFree(p->colVal); + taosMemoryFree(p); } SIndexMultiTerm* indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm*)); } @@ -536,7 +536,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { taosArrayClear(value->val); } } - free(value->colVal); + taosMemoryFree(value->colVal); value->colVal = NULL; } static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { @@ -572,7 +572,7 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { END: if (tw != NULL) { writerCtxDestroy(tw->ctx, true); - free(tw); + taosMemoryFree(tw); } return -1; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 2b4327b091..bf907726bc 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -40,7 +40,7 @@ static bool indexCacheIteratorNext(Iterate* itera); static IterateValue* indexCacheIteratorGetValue(Iterate* iter); IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { - IndexCache* cache = calloc(1, sizeof(IndexCache)); + IndexCache* cache = taosMemoryCalloc(1, sizeof(IndexCache)); if (cache == NULL) { indexError("failed to create index cache"); return NULL; @@ -113,8 +113,8 @@ void indexCacheDestroySkiplist(SSkipList* slt) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); if (ct != NULL) { - free(ct->colVal); - free(ct); + taosMemoryFree(ct->colVal); + taosMemoryFree(ct); } } tSkipListDestroyIter(iter); @@ -144,16 +144,16 @@ void indexCacheDestroy(void* cache) { } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); - free(pCache->colName); + taosMemoryFree(pCache->colName); taosThreadMutexDestroy(&pCache->mtx); taosThreadCondDestroy(&pCache->finished); - free(pCache); + taosMemoryFree(pCache); } Iterate* indexCacheIteratorCreate(IndexCache* cache) { - Iterate* iiter = calloc(1, sizeof(Iterate)); + Iterate* iiter = taosMemoryCalloc(1, sizeof(Iterate)); if (iiter == NULL) { return NULL; } @@ -179,7 +179,7 @@ void indexCacheIteratorDestroy(Iterate* iter) { } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); - free(iter); + taosMemoryFree(iter); } int indexCacheSchedToMerge(IndexCache* pCache) { @@ -221,7 +221,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { IndexCache* pCache = cache; indexCacheRef(pCache); // encode data - CacheTerm* ct = calloc(1, sizeof(CacheTerm)); + CacheTerm* ct = taosMemoryCalloc(1, sizeof(CacheTerm)); if (cache == NULL) { return -1; } @@ -230,7 +230,7 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { if (hasJson) { ct->colVal = indexPackJsonData(term); } else { - ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); + ct->colVal = (char*)taosMemoryCalloc(1, sizeof(char) * (term->nColVal + 1)); memcpy(ct->colVal, term->colVal, term->nColVal); } ct->version = atomic_add_fetch_32(&pCache->version, 1); @@ -323,7 +323,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result } if (hasJson) { - tfree(p); + taosMemoryFreeClear(p); } indexMemUnRef(mem); @@ -365,7 +365,7 @@ void indexMemUnRef(MemTable* tbl) { if (ref == 0) { SSkipList* slt = tbl->mem; indexCacheDestroySkiplist(slt); - free(tbl); + taosMemoryFree(tbl); } } @@ -373,8 +373,8 @@ static void indexCacheTermDestroy(CacheTerm* ct) { if (ct == NULL) { return; } - free(ct->colVal); - free(ct); + taosMemoryFree(ct->colVal); + taosMemoryFree(ct); } static char* indexCacheTermGet(const void* pData) { CacheTerm* p = (CacheTerm*)pData; @@ -394,7 +394,7 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) { static MemTable* indexInternalCacheCreate(int8_t type) { type = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : type; - MemTable* tbl = calloc(1, sizeof(MemTable)); + MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); indexMemRef(tbl); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, indexCacheTermCompare, SL_ALLOW_DUP_KEY, diff --git a/source/libs/index/src/index_comm.c b/source/libs/index/src/index_comm.c index 4f3cbaa4da..4dea5fa011 100644 --- a/source/libs/index/src/index_comm.c +++ b/source/libs/index/src/index_comm.c @@ -27,7 +27,7 @@ char* indexPackJsonData(SIndexTerm* itm) { uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; - char* buf = (char*)calloc(1, sz); + char* buf = (char*)taosMemoryCalloc(1, sz); char* p = buf; memcpy(p, itm->colName, itm->nColName); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 58d5a871cd..09f382bbdc 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -30,7 +30,7 @@ static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, Compi } FstUnFinishedNodes* fstUnFinishedNodesCreate() { - FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); + FstUnFinishedNodes* nodes = taosMemoryMalloc(sizeof(FstUnFinishedNodes)); if (nodes == NULL) { return NULL; } @@ -42,7 +42,7 @@ FstUnFinishedNodes* fstUnFinishedNodesCreate() { static void unFinishedNodeDestroyElem(void* elem) { FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; fstBuilderNodeDestroy(b->node); - free(b->last); + taosMemoryFree(b->last); b->last = NULL; } void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { @@ -51,11 +51,11 @@ void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { } taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); - free(nodes); + taosMemoryFree(nodes); } void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { - FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); node->isFinal = isFinal; node->finalOutput = 0; node->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -74,7 +74,7 @@ FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); fstBuilderNodeUnfinishedLastCompiled(un, addr); - // free(un->last); // TODO add func FstLastTransitionFree() + // taosMemoryFree(un->last); // TODO add func FstLastTransitionFree() // un->last = NULL; return un->node; } @@ -103,7 +103,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); - // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); // trn->inp = s->data[s->start]; // trn->out = out; int32_t len = 0; @@ -111,12 +111,12 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output un->last = fstLastTransitionCreate(data[0], out); for (uint64_t i = 1; i < len; i++) { - FstBuilderNode* n = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* n = taosMemoryMalloc(sizeof(FstBuilderNode)); n->isFinal = false; n->finalOutput = 0; n->trans = taosArrayInit(16, sizeof(FstTransition)); - // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); // trn->inp = s->data[i]; // trn->out = out; FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); @@ -296,7 +296,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil // at that index. (Except when there are 256 transitions.) Namely, // any value greater than or equal to the number of transitions in // this node indicates an absent transition. - uint8_t* index = (uint8_t*)malloc(sizeof(uint8_t) * 256); + uint8_t* index = (uint8_t*)taosMemoryMalloc(sizeof(uint8_t) * 256); memset(index, 255, sizeof(uint8_t) * 256); /// for (uint8_t i = 0; i < 256; i++) { // index[i] = 255; @@ -307,7 +307,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil // fstPackDeltaIn(w, addr, t->addr, tSize); } fstCountingWriterWrite(w, (char*)index, 256); - free(index); + taosMemoryFree(index); } fstCountingWriterWrite(w, (char*)&packSizes, 1); bool null = false; @@ -578,7 +578,7 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { // fst node function FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { - FstNode* n = (FstNode*)malloc(sizeof(FstNode)); + FstNode* n = (FstNode*)taosMemoryMalloc(sizeof(FstNode)); if (n == NULL) { return NULL; } @@ -643,10 +643,10 @@ static const char* fstNodeState(FstNode* node) { void fstNodeDestroy(FstNode* node) { fstSliceDestroy(&node->data); - free(node); + taosMemoryFree(node); } FstTransitions* fstNodeTransitions(FstNode* node) { - FstTransitions* t = malloc(sizeof(FstTransitions)); + FstTransitions* t = taosMemoryMalloc(sizeof(FstTransitions)); if (NULL == t) { return NULL; } @@ -755,7 +755,7 @@ bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, Compiled } FstBuilder* fstBuilderCreate(void* w, FstType ty) { - FstBuilder* b = malloc(sizeof(FstBuilder)); + FstBuilder* b = taosMemoryMalloc(sizeof(FstBuilder)); if (NULL == b) { return b; } @@ -788,7 +788,7 @@ void fstBuilderDestroy(FstBuilder* b) { fstUnFinishedNodesDestroy(b->unfinished); fstRegistryDestroy(b->registry); fstSliceDestroy(&b->last); - free(b); + taosMemoryFree(b); } bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { @@ -928,7 +928,7 @@ FstSlice fstNodeAsSlice(FstNode* node) { } FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { - FstLastTransition* trn = malloc(sizeof(FstLastTransition)); + FstLastTransition* trn = taosMemoryMalloc(sizeof(FstLastTransition)); if (trn == NULL) { return NULL; } @@ -938,7 +938,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { return trn; } -void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } +void fstLastTransitionDestroy(FstLastTransition* trn) { taosMemoryFree(trn); } void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; @@ -1003,12 +1003,12 @@ Fst* fstCreate(FstSlice* slice) { len -= sizeof(fstLen); taosDecodeFixedU64(buf + len, &fstLen); // TODO(validate root addr) - Fst* fst = (Fst*)calloc(1, sizeof(Fst)); + Fst* fst = (Fst*)taosMemoryCalloc(1, sizeof(Fst)); if (fst == NULL) { return NULL; } - fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); + fst->meta = (FstMeta*)taosMemoryMalloc(sizeof(FstMeta)); if (NULL == fst->meta) { goto FST_CREAT_FAILED; } @@ -1019,7 +1019,7 @@ Fst* fstCreate(FstSlice* slice) { fst->meta->len = fstLen; fst->meta->checkSum = checkSum; - FstSlice* s = calloc(1, sizeof(FstSlice)); + FstSlice* s = taosMemoryCalloc(1, sizeof(FstSlice)); *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); fst->data = s; @@ -1027,19 +1027,19 @@ Fst* fstCreate(FstSlice* slice) { return fst; FST_CREAT_FAILED: - free(fst->meta); - free(fst); + taosMemoryFree(fst->meta); + taosMemoryFree(fst); return NULL; } void fstDestroy(Fst* fst) { if (fst) { - free(fst->meta); + taosMemoryFree(fst->meta); fstSliceDestroy(fst->data); - free(fst->data); + taosMemoryFree(fst->data); taosThreadMutexDestroy(&fst->mtx); } - free(fst); + taosMemoryFree(fst); } bool fstGet(Fst* fst, FstSlice* b, Output* out) { @@ -1136,7 +1136,7 @@ bool fstVerify(Fst* fst) { // data bound function FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { - FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); + FstBoundWithData* b = taosMemoryCalloc(1, sizeof(FstBoundWithData)); if (b == NULL) { return NULL; } @@ -1171,11 +1171,11 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type == Excluded ? false : true; } -void fstBoundDestroy(FstBoundWithData* bound) { free(bound); } +void fstBoundDestroy(FstBoundWithData* bound) { taosMemoryFree(bound); } StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { - StreamWithState* sws = calloc(1, sizeof(StreamWithState)); + StreamWithState* sws = taosMemoryCalloc(1, sizeof(StreamWithState)); if (sws == NULL) { return NULL; } @@ -1201,7 +1201,7 @@ void streamWithStateDestroy(StreamWithState* sws) { taosArrayDestroy(sws->inp); taosArrayDestroyEx(sws->stack, streamStateDestroy); - free(sws); + taosMemoryFree(sws); } bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { @@ -1346,7 +1346,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb taosArrayPush(sws->stack, &s2); size_t isz = taosArrayGetSize(sws->inp); - uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); + uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t)); for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } @@ -1354,19 +1354,19 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); - tfree(buf); + taosMemoryFreeClear(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); - tfree(buf); + taosMemoryFreeClear(buf); fstSliceDestroy(&slice); taosArrayDestroy(nodes); return result; } - tfree(buf); + taosMemoryFreeClear(buf); fstSliceDestroy(&slice); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { @@ -1378,7 +1378,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb } StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { - StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); + StreamWithStateResult* result = taosMemoryCalloc(1, sizeof(StreamWithStateResult)); if (result == NULL) { return NULL; } @@ -1395,7 +1395,7 @@ void swsResultDestroy(StreamWithStateResult* result) { fstSliceDestroy(&result->data); startWithStateValueDestroy(result->state); - free(result); + taosMemoryFree(result); } void streamStateDestroy(void* s) { @@ -1407,7 +1407,7 @@ void streamStateDestroy(void* s) { } FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { - FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); + FstStreamBuilder* b = taosMemoryCalloc(1, sizeof(FstStreamBuilder)); if (NULL == b) { return NULL; } @@ -1421,9 +1421,9 @@ FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { void fstStreamBuilderDestroy(FstStreamBuilder* b) { fstSliceDestroy(&b->min->data); fstSliceDestroy(&b->max->data); - tfree(b->min); - tfree(b->max); - free(b); + taosMemoryFreeClear(b->min); + taosMemoryFreeClear(b->max); + taosMemoryFree(b); } FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { if (b == NULL) { diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index ed1ad7a374..668a527d4a 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,7 +16,7 @@ #include "index_fst_automation.h" StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { - StartWithStateValue* sv = calloc(1, sizeof(StartWithStateValue)); + StartWithStateValue* sv = taosMemoryCalloc(1, sizeof(StartWithStateValue)); if (sv == NULL) { return NULL; } @@ -27,7 +27,7 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp sv->val = *(int*)val; } else if (ty == FST_CHAR) { size_t len = strlen((char*)val); - sv->ptr = (char*)calloc(1, len + 1); + sv->ptr = (char*)taosMemoryCalloc(1, len + 1); memcpy(sv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, @@ -44,14 +44,14 @@ void startWithStateValueDestroy(void* val) { if (sv->type == FST_INT) { // } else if (sv->type == FST_CHAR) { - free(sv->ptr); + taosMemoryFree(sv->ptr); } else if (sv->type == FST_ARRAY) { taosArrayDestroy(sv->arr); } - free(sv); + taosMemoryFree(sv); } StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { - StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); + StartWithStateValue* nsv = taosMemoryCalloc(1, sizeof(StartWithStateValue)); if (nsv == NULL) { return NULL; } @@ -62,7 +62,7 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { nsv->val = sv->val; } else if (nsv->type == FST_CHAR) { size_t len = strlen(sv->ptr); - nsv->ptr = (char*)calloc(1, len + 1); + nsv->ptr = (char*)taosMemoryCalloc(1, len + 1); memcpy(nsv->ptr, sv->ptr, len); } else if (nsv->type == FST_ARRAY) { // @@ -137,7 +137,7 @@ AutomationFunc automFuncs[] = { }; AutomationCtx* automCtxCreate(void* data, AutomationType atype) { - AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); + AutomationCtx* ctx = taosMemoryCalloc(1, sizeof(AutomationCtx)); if (ctx == NULL) { return NULL; } @@ -158,7 +158,7 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (data != NULL) { char* src = (char*)data; size_t len = strlen(src); - dst = (char*)calloc(1, len * sizeof(char) + 1); + dst = (char*)taosMemoryCalloc(1, len * sizeof(char) + 1); memcpy(dst, src, len); } @@ -169,6 +169,6 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { } void automCtxDestroy(AutomationCtx* ctx) { startWithStateValueDestroy(ctx->stdata); - free(ctx->data); - free(ctx); + taosMemoryFree(ctx->data); + taosMemoryFree(ctx); } diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 8cb2ff9246..6a161ba7e9 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -81,7 +81,7 @@ static int writeCtxDoFlush(WriterCtx* ctx) { } WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) { - WriterCtx* ctx = calloc(1, sizeof(WriterCtx)); + WriterCtx* ctx = taosMemoryCalloc(1, sizeof(WriterCtx)); if (ctx == NULL) { return NULL; } ctx->type = type; @@ -112,7 +112,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int goto END; } } else if (ctx->type == TMemory) { - ctx->mem.buf = calloc(1, sizeof(char) * capacity); + ctx->mem.buf = taosMemoryCalloc(1, sizeof(char) * capacity); ctx->mem.capa = capacity; } ctx->write = writeCtxDoWrite; @@ -126,13 +126,13 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int return ctx; END: - if (ctx->type == TMemory) { free(ctx->mem.buf); } - free(ctx); + if (ctx->type == TMemory) { taosMemoryFree(ctx->mem.buf); } + taosMemoryFree(ctx); return NULL; } void writerCtxDestroy(WriterCtx* ctx, bool remove) { if (ctx->type == TMemory) { - free(ctx->mem.buf); + taosMemoryFree(ctx->mem.buf); } else { ctx->flush(ctx); taosCloseFile(&ctx->file.pFile); @@ -150,11 +150,11 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { } if (remove) { unlink(ctx->file.buf); } } - free(ctx); + taosMemoryFree(ctx); } FstCountingWriter* fstCountingWriterCreate(void* wrt) { - FstCountingWriter* cw = calloc(1, sizeof(FstCountingWriter)); + FstCountingWriter* cw = taosMemoryCalloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } cw->wrt = wrt; @@ -165,7 +165,7 @@ void fstCountingWriterDestroy(FstCountingWriter* cw) { // free wrt object: close fd or free mem fstCountingWriterFlush(cw); // writerCtxDestroy((WriterCtx *)(cw->wrt)); - free(cw); + taosMemoryFree(cw); } int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) { @@ -203,13 +203,13 @@ int fstCountingWriterFlush(FstCountingWriter* write) { void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes) { assert(1 <= nBytes && nBytes <= 8); - uint8_t* buf = calloc(8, sizeof(uint8_t)); + uint8_t* buf = taosMemoryCalloc(8, sizeof(uint8_t)); for (uint8_t i = 0; i < nBytes; i++) { buf[i] = (uint8_t)n; n = n >> 8; } fstCountingWriterWrite(writer, buf, nBytes); - free(buf); + taosMemoryFree(buf); return; } diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index 5cdf6adb31..a0d59150d7 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -15,7 +15,7 @@ #include "index_fst_node.h" FstBuilderNode* fstBuilderNodeDefault() { - FstBuilderNode* bn = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* bn = taosMemoryMalloc(sizeof(FstBuilderNode)); bn->isFinal = false; bn->finalOutput = 0; bn->trans = taosArrayInit(16, sizeof(FstTransition)); @@ -25,7 +25,7 @@ void fstBuilderNodeDestroy(FstBuilderNode* node) { if (node == NULL) { return; } taosArrayDestroy(node->trans); - free(node); + taosMemoryFree(node); } bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { @@ -45,7 +45,7 @@ bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { return true; } FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) { - FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); + FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); if (node == NULL) { return NULL; } // diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index b28b518fc1..7b6b8df25f 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -65,13 +65,13 @@ static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) { } FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { - FstRegistry* registry = malloc(sizeof(FstRegistry)); + FstRegistry* registry = taosMemoryMalloc(sizeof(FstRegistry)); if (registry == NULL) { return NULL; } uint64_t nCells = tableSize * mruSize; SArray* tb = (SArray*)taosArrayInit(nCells, sizeof(FstRegistryCell)); if (NULL == tb) { - free(registry); + taosMemoryFree(registry); return NULL; } @@ -96,7 +96,7 @@ void fstRegistryDestroy(FstRegistry* registry) { fstBuilderNodeDestroy(cell->node); } taosArrayDestroy(tb); - free(registry); + taosMemoryFree(registry); } FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode) { @@ -105,7 +105,7 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo uint64_t start = registry->mruSize * bucket; uint64_t end = start + registry->mruSize; - FstRegistryEntry* entry = malloc(sizeof(FstRegistryEntry)); + FstRegistryEntry* entry = taosMemoryMalloc(sizeof(FstRegistryEntry)); if (end - start == 1) { FstRegistryCell* cell = taosArrayGet(registry->table, start); // cell->isNode && @@ -166,5 +166,5 @@ FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNo return entry; } void fstRegistryEntryDestroy(FstRegistryEntry* entry) { - free(entry); + taosMemoryFree(entry); } diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index f08a48c34e..f9581f7202 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -78,10 +78,10 @@ CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) { // FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { - FstString* str = (FstString*)malloc(sizeof(FstString)); + FstString* str = (FstString*)taosMemoryMalloc(sizeof(FstString)); str->ref = 1; str->len = len; - str->data = malloc(len * sizeof(uint8_t)); + str->data = taosMemoryMalloc(len * sizeof(uint8_t)); memcpy(str->data, data, len); FstSlice s = {.str = str, .start = 0, .end = len - 1}; @@ -101,10 +101,10 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { uint8_t* data = fstSliceData(s, &slen); assert(tlen <= slen); - uint8_t* buf = malloc(sizeof(uint8_t) * tlen); + uint8_t* buf = taosMemoryMalloc(sizeof(uint8_t) * tlen); memcpy(buf, data + start, tlen); - FstString* str = malloc(sizeof(FstString)); + FstString* str = taosMemoryMalloc(sizeof(FstString)); str->data = buf; str->len = tlen; str->ref = 1; @@ -128,8 +128,8 @@ void fstSliceDestroy(FstSlice* s) { FstString* str = s->str; str->ref--; if (str->ref == 0) { - free(str->data); - free(str); + taosMemoryFree(str->data); + taosMemoryFree(str); s->str = NULL; } } @@ -161,7 +161,7 @@ int fstSliceCompare(FstSlice* a, FstSlice* b) { } // FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) { -// FstStack *s = calloc(1, sizeof(FstStack)); +// FstStack *s = taosMemoryCalloc(1, sizeof(FstStack)); // if (s == NULL) { return NULL; } // s-> // s->freeFn diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 780b7160fc..53813e13e6 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -59,7 +59,7 @@ static void tfileGenFileName(char* filename, uint64_t suid, const char* col, static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version); TFileCache* tfileCacheCreate(const char* path) { - TFileCache* tcache = calloc(1, sizeof(TFileCache)); + TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); if (tcache == NULL) { return NULL; } @@ -113,7 +113,7 @@ void tfileCacheDestroy(TFileCache* tcache) { reader = taosHashIterate(tcache->tableCache, reader); } taosHashCleanup(tcache->tableCache); - free(tcache); + taosMemoryFree(tcache); } TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { @@ -145,7 +145,7 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { return; } TFileReader* tfileReaderCreate(WriterCtx* ctx) { - TFileReader* reader = calloc(1, sizeof(TFileReader)); + TFileReader* reader = taosMemoryCalloc(1, sizeof(TFileReader)); if (reader == NULL) { return NULL; } @@ -181,7 +181,7 @@ void tfileReaderDestroy(TFileReader* reader) { // T_REF_INC(reader); fstDestroy(reader->fst); writerCtxDestroy(reader->ctx, reader->remove); - free(reader); + taosMemoryFree(reader); } int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { @@ -218,7 +218,7 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResul } fstSliceDestroy(&key); if (hasJson) { - free(p); + taosMemoryFree(p); } } else if (qtype == QUERY_PREFIX) { // handle later @@ -269,7 +269,7 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c return reader; } TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { - TFileWriter* tw = calloc(1, sizeof(TFileWriter)); + TFileWriter* tw = taosMemoryCalloc(1, sizeof(TFileWriter)); if (tw == NULL) { indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid); return NULL; @@ -296,7 +296,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { } int32_t bufLimit = 64 * 4096, offset = 0; - // char* buf = calloc(1, sizeof(char) * bufLimit); + // char* buf = taosMemoryCalloc(1, sizeof(char) * bufLimit); // char* p = buf; int32_t sz = taosArrayGetSize((SArray*)data); int32_t fstOffset = tw->offset; @@ -318,13 +318,13 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { // check buf has enough space or not int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); - char* buf = calloc(1, ttsz * sizeof(char)); + char* buf = taosMemoryCalloc(1, ttsz * sizeof(char)); char* p = buf; tfileSerialTableIdsToBuf(p, v->tableId); tw->ctx->write(tw->ctx, buf, ttsz); v->offset = tw->offset; tw->offset += ttsz; - free(buf); + taosMemoryFree(buf); } tw->fb = fstBuilderCreate(tw->ctx, 0); @@ -359,14 +359,14 @@ void tfileWriterClose(TFileWriter* tw) { return; } writerCtxDestroy(tw->ctx, false); - free(tw); + taosMemoryFree(tw); } void tfileWriterDestroy(TFileWriter* tw) { if (tw == NULL) { return; } writerCtxDestroy(tw->ctx, false); - free(tw); + taosMemoryFree(tw); } IndexTFile* indexTFileCreate(const char* path) { @@ -375,7 +375,7 @@ IndexTFile* indexTFileCreate(const char* path) { return NULL; } - IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); + IndexTFile* tfile = taosMemoryCalloc(1, sizeof(IndexTFile)); if (tfile == NULL) { tfileCacheDestroy(cache); return NULL; @@ -389,7 +389,7 @@ void indexTFileDestroy(IndexTFile* tfile) { return; } tfileCacheDestroy(tfile->cache); - free(tfile); + taosMemoryFree(tfile); } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTempResult* result) { @@ -433,7 +433,7 @@ static bool tfileIteratorNext(Iterate* iiter) { int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); - colVal = calloc(1, sz + 1); + colVal = taosMemoryCalloc(1, sz + 1); memcpy(colVal, ch, sz); offset = (uint64_t)(rt->out.out); @@ -453,7 +453,7 @@ static bool tfileIteratorNext(Iterate* iiter) { static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { - TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); + TFileFstIter* tIter = taosMemoryCalloc(1, sizeof(TFileFstIter)); if (tIter == NULL) { return NULL; } @@ -470,10 +470,10 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { return NULL; } - Iterate* iter = calloc(1, sizeof(Iterate)); + Iterate* iter = taosMemoryCalloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); if (iter->iter == NULL) { - free(iter); + taosMemoryFree(iter); return NULL; } iter->next = tfileIteratorNext; @@ -494,9 +494,9 @@ void tfileIteratorDestroy(Iterate* iter) { streamWithStateDestroy(tIter->st); fstStreamBuilderDestroy(tIter->fb); automCtxDestroy(tIter->ctx); - free(tIter); + taosMemoryFree(tIter); - free(iter); + taosMemoryFree(iter); } TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { @@ -530,7 +530,7 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { } TFileValue* tfileValueCreate(char* val) { - TFileValue* tf = calloc(1, sizeof(TFileValue)); + TFileValue* tf = taosMemoryCalloc(1, sizeof(TFileValue)); if (tf == NULL) { return NULL; } @@ -547,8 +547,8 @@ int tfileValuePush(TFileValue* tf, uint64_t val) { } void tfileValueDestroy(TFileValue* tf) { taosArrayDestroy(tf->tableId); - free(tf->colVal); - free(tf); + taosMemoryFree(tf->colVal); + taosMemoryFree(tf); } static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { int sz = taosArrayGetSize(ids); @@ -636,7 +636,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { // current load fst into memory, refactor it later int fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber); - char* buf = calloc(1, fstSize); + char* buf = taosMemoryCalloc(1, fstSize); if (buf == NULL) { return -1; } @@ -651,7 +651,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { FstSlice st = fstSliceCreate((uint8_t*)buf, nread); reader->fst = fstCreate(&st); - free(buf); + taosMemoryFree(buf); fstSliceDestroy(&st); return reader->fst != NULL ? 0 : -1; @@ -744,7 +744,7 @@ static SArray* tfileGetFileList(const char* path) { } size_t len = strlen(path) + 1 + strlen(file) + 1; - char* buf = calloc(1, len); + char* buf = taosMemoryCalloc(1, len); sprintf(buf, "%s/%s", path, file); taosArrayPush(files, &buf); } @@ -761,7 +761,7 @@ static int tfileRmExpireFile(SArray* result) { } static void tfileDestroyFileName(void* elem) { char* p = *(char**)elem; - free(p); + taosMemoryFree(p); } static int tfileCompare(const void* a, const void* b) { const char* as = *(char**)a; diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c index dfe4e273a9..65c16ca65b 100644 --- a/source/libs/index/src/index_util.c +++ b/source/libs/index/src/index_util.c @@ -41,7 +41,7 @@ void iIntersection(SArray *inters, SArray *final) { if (sz <= 0) { return; } - MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); for (int i = 0; i < sz; i++) { SArray *t = taosArrayGetP(inters, i); mi[i].len = taosArrayGetSize(t); @@ -67,7 +67,7 @@ void iIntersection(SArray *inters, SArray *final) { taosArrayPush(final, &tgt); } } - tfree(mi); + taosMemoryFreeClear(mi); } void iUnion(SArray *inters, SArray *final) { int32_t sz = taosArrayGetSize(inters); @@ -79,7 +79,7 @@ void iUnion(SArray *inters, SArray *final) { return; } - MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + MergeIndex *mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); for (int i = 0; i < sz; i++) { SArray *t = taosArrayGetP(inters, i); mi[i].len = taosArrayGetSize(t); @@ -113,7 +113,7 @@ void iUnion(SArray *inters, SArray *final) { break; } } - tfree(mi); + taosMemoryFreeClear(mi); } void iExcept(SArray *total, SArray *except) { @@ -156,7 +156,7 @@ int verdataCompare(const void *a, const void *b) { } SIdxTempResult *sIdxTempResultCreate() { - SIdxTempResult *tr = calloc(1, sizeof(SIdxTempResult)); + SIdxTempResult *tr = taosMemoryCalloc(1, sizeof(SIdxTempResult)); tr->total = taosArrayInit(4, sizeof(uint64_t)); tr->added = taosArrayInit(4, sizeof(uint64_t)); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 410d5b7021..94923726dd 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -55,7 +55,7 @@ class FstReadMemory { memset((void*)&_s, 0, sizeof(_s)); } bool init() { - char* buf = (char*)calloc(1, sizeof(char) * _size); + char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); if (nRead <= 0) { return false; @@ -63,7 +63,7 @@ class FstReadMemory { _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); - free(buf); + taosMemoryFree(buf); return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { @@ -230,7 +230,7 @@ void checkFstLongTerm() { // for (int i = 0; i < result.size(); i++) { // assert(result[i] == i); // check result //} - // free(ctx); + // taosMemoryFree(ctx); // delete m; } void checkFstCheckIterator() { @@ -266,7 +266,7 @@ void checkFstCheckIterator() { // assert(result[i] == i); // check result } - free(ctx); + taosMemoryFree(ctx); delete m; } diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index 6e9c88bc79..1bdc7fc9c9 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -75,7 +75,7 @@ class FstReadMemory { memset((void*)&_s, 0, sizeof(_s)); } bool init() { - char* buf = (char*)calloc(1, sizeof(char) * _size); + char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); if (nRead <= 0) { return false; @@ -83,7 +83,7 @@ class FstReadMemory { _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); - free(buf); + taosMemoryFreeClear(buf); return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 5f339f2865..0e4eb060cf 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -81,7 +81,7 @@ class FstReadMemory { memset((void*)&_s, 0, sizeof(_s)); } bool init() { - char* buf = (char*)calloc(1, sizeof(char) * _size); + char* buf = (char*)taosMemoryCalloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); if (nRead <= 0) { return false; @@ -89,7 +89,7 @@ class FstReadMemory { _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); - free(buf); + taosMemoryFree(buf); return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { @@ -227,7 +227,7 @@ void checkFstPrefixSearch() { assert(result[i] == i); // check result } - free(ctx); + taosMemoryFree(ctx); delete m; } void validateFst() { @@ -446,9 +446,9 @@ class IndexTFileEnv : public ::testing::Test { }; static TFileValue* genTFileValue(const char* val) { - TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue)); + TFileValue* tv = (TFileValue*)taosMemoryCalloc(1, sizeof(TFileValue)); int32_t vlen = strlen(val) + 1; - tv->colVal = (char*)calloc(1, vlen); + tv->colVal = (char*)taosMemoryCalloc(1, vlen); memcpy(tv->colVal, val, vlen); tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -460,9 +460,9 @@ static TFileValue* genTFileValue(const char* val) { } static void destroyTFileValue(void* val) { TFileValue* tv = (TFileValue*)val; - free(tv->colVal); + taosMemoryFree(tv->colVal); taosArrayDestroy(tv->tableId); - free(tv); + taosMemoryFree(tv); } TEST_F(IndexTFileEnv, test_tfile_write) { TFileValue* v1 = genTFileValue("ab"); diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index aa5eafd8b2..b92c08d51c 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -60,7 +60,7 @@ void monCleanup() { } SMonInfo *monCreateMonitorInfo() { - SMonInfo *pMonitor = calloc(1, sizeof(SMonInfo)); + SMonInfo *pMonitor = taosMemoryCalloc(1, sizeof(SMonInfo)); if (pMonitor == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -88,7 +88,7 @@ void monCleanupMonitorInfo(SMonInfo *pMonitor) { tsMonitor.state.time = pMonitor->curTime; taosArrayDestroy(pMonitor->logs); tjsonDelete(pMonitor->pJson); - free(pMonitor); + taosMemoryFree(pMonitor); } void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { @@ -381,6 +381,6 @@ void monSendReport(SMonInfo *pMonitor) { if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.comp ? HTTP_GZIP : HTTP_FLAT; taosSendHttpReport(tsMonitor.server, tsMonitor.port, pCont, strlen(pCont), flag); - free(pCont); + taosMemoryFree(pCont); } } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 12cd6f1bc1..9d0b79ca80 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -19,6 +19,11 @@ #include "taos.h" #include "taoserror.h" +#define COPY_ALL_SCALAR_FIELDS \ + do { \ + memcpy((pDst), (pSrc), sizeof(*pSrc)); \ + } while (0) + #define COPY_SCALAR_FIELD(fldname) \ do { \ (pDst)->fldname = (pSrc)->fldname; \ @@ -141,12 +146,12 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - pDst->datum.p = malloc(pSrc->node.resType.bytes + VARSTR_HEADER_SIZE); + pDst->datum.p = taosMemoryMalloc(pSrc->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pDst->datum.p) { nodesDestroyNode(pDst); return NULL; } - memcpy(pDst->datum.p, pSrc->datum.p, pSrc->node.resType.bytes + VARSTR_HEADER_SIZE); + memcpy(pDst->datum.p, pSrc->datum.p, pSrc->node.resType.bytes + VARSTR_HEADER_SIZE + 1); break; case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: @@ -195,6 +200,12 @@ static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode return (SNode*)pDst; } +static SNode* orderByExprNodeCopy(const SOrderByExprNode* pSrc, SOrderByExprNode* pDst) { + COPY_ALL_SCALAR_FIELDS; + CLONE_NODE_FIELD(pExpr); + return (SNode*)pDst; +} + static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pValues); @@ -210,7 +221,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { static STableMeta* tableMetaClone(const STableMeta* pSrc) { int32_t len = TABLE_META_SIZE(pSrc); - STableMeta* pDst = malloc(len); + STableMeta* pDst = taosMemoryMalloc(len); if (NULL == pDst) { return NULL; } @@ -220,7 +231,7 @@ static STableMeta* tableMetaClone(const STableMeta* pSrc) { static SVgroupsInfo* vgroupsInfoClone(const SVgroupsInfo* pSrc) { int32_t len = VGROUPS_INFO_SIZE(pSrc); - SVgroupsInfo* pDst = malloc(len); + SVgroupsInfo* pDst = taosMemoryMalloc(len); if (NULL == pDst) { return NULL; } @@ -237,6 +248,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(scanFlag); COPY_SCALAR_FIELD(scanRange); COPY_SCALAR_FIELD(tableName); + COPY_SCALAR_FIELD(showRewrite); return (SNode*)pDst; } @@ -250,6 +262,7 @@ static SNode* logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pProjections); + COPY_CHAR_ARRAY_FIELD(stmtName); return (SNode*)pDst; } @@ -266,16 +279,24 @@ static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNo } static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pDst) { + COPY_ALL_SCALAR_FIELDS; COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); - COPY_SCALAR_FIELD(winType); + // COPY_SCALAR_FIELD(winType); CLONE_NODE_LIST_FIELD(pFuncs); - COPY_SCALAR_FIELD(interval); - COPY_SCALAR_FIELD(offset); - COPY_SCALAR_FIELD(sliding); - COPY_SCALAR_FIELD(intervalUnit); - COPY_SCALAR_FIELD(slidingUnit); + // COPY_SCALAR_FIELD(interval); + // COPY_SCALAR_FIELD(offset); + // COPY_SCALAR_FIELD(sliding); + // COPY_SCALAR_FIELD(intervalUnit); + // COPY_SCALAR_FIELD(slidingUnit); CLONE_NODE_FIELD(pFill); - COPY_SCALAR_FIELD(sessionGap); + // COPY_SCALAR_FIELD(sessionGap); + CLONE_NODE_FIELD(pTspk); + return (SNode*)pDst; +} + +static SNode* logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + CLONE_NODE_LIST_FIELD(pSortKeys); return (SNode*)pDst; } @@ -338,6 +359,7 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { case QUERY_NODE_GROUPING_SET: return groupingSetNodeCopy((const SGroupingSetNode*)pNode, (SGroupingSetNode*)pDst); case QUERY_NODE_ORDER_BY_EXPR: + return orderByExprNodeCopy((const SOrderByExprNode*)pNode, (SOrderByExprNode*)pDst); case QUERY_NODE_LIMIT: break; case QUERY_NODE_FILL: @@ -360,6 +382,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_WINDOW: return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_SORT: + return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst); case QUERY_NODE_LOGIC_SUBPLAN: return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst); default: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b648e18809..15e03b9891 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -764,6 +764,8 @@ static int32_t jsonToEpSet(const SJson* pJson, void* pObj) { } static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet"; +static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite"; +static const char* jkSysTableScanPhysiPlanAccountId = "AccountId"; static int32_t physiSysTableScanNodeToJson(const void* pObj, SJson* pJson) { const SSystemTableScanPhysiNode* pNode = (const SSystemTableScanPhysiNode*)pObj; @@ -772,6 +774,12 @@ static int32_t physiSysTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSysTableScanPhysiPlanMnodeEpSet, epSetToJson, &pNode->mgmtEpSet); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSysTableScanPhysiPlanShowRewrite, pNode->showRewrite); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId); + } return code; } @@ -783,6 +791,12 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkSysTableScanPhysiPlanMnodeEpSet, jsonToEpSet, &pNode->mgmtEpSet); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSysTableScanPhysiPlanShowRewrite, &pNode->showRewrite); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkSysTableScanPhysiPlanAccountId, pNode->accountId); + } return code; } @@ -837,9 +851,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - int32_t val; - code = tjsonGetIntValue(pJson, jkJoinPhysiPlanJoinType, &val); - pNode->joinType = val; + code = tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType); } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions); @@ -920,6 +932,37 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkSortPhysiPlanExprs = "Exprs"; +static const char* jkSortPhysiPlanSortKeys = "SortKeys"; + +static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) { + const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkSortPhysiPlanExprs, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkSortPhysiPlanSortKeys, pNode->pSortKeys); + } + + return code; +} + +static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) { + SSortPhysiNode* pNode = (SSortPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSortPhysiPlanExprs, &pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSortPhysiPlanSortKeys, &pNode->pSortKeys); + } + + return code; +} + static const char* jkWindowPhysiPlanExprs = "Exprs"; static const char* jkWindowPhysiPlanFuncs = "Funcs"; @@ -957,6 +1000,7 @@ static const char* jkIntervalPhysiPlanSliding = "Sliding"; static const char* jkIntervalPhysiPlanIntervalUnit = "intervalUnit"; static const char* jkIntervalPhysiPlanSlidingUnit = "slidingUnit"; static const char* jkIntervalPhysiPlanFill = "Fill"; +static const char* jkIntervalPhysiPlanTsPk = "TsPk"; static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; @@ -980,6 +1024,9 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkIntervalPhysiPlanFill, nodeToJson, pNode->pFill); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalPhysiPlanTsPk, nodeToJson, pNode->pTspk); + } return code; } @@ -1006,6 +1053,9 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkIntervalPhysiPlanFill, (SNode**)&pNode->pFill); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalPhysiPlanTsPk, (SNode**)&pNode->pTspk); + } return code; } @@ -1164,9 +1214,7 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { int32_t code = tjsonToObject(pJson, jkSubplanId, jsonToSubplanId, &pNode->id); if (TSDB_CODE_SUCCESS == code) { - int32_t val; - code = tjsonGetIntValue(pJson, jkSubplanType, &val); - pNode->subplanType = val; + code = tjsonGetNumberValue(pJson, jkSubplanType, pNode->subplanType); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanMsgType, &pNode->msgType); @@ -1356,9 +1404,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId); } if (TSDB_CODE_SUCCESS == code) { - int32_t tmp; - code = tjsonGetIntValue(pJson, jkColumnColType, &tmp); - pNode->colType = tmp; + code = tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName); @@ -1480,7 +1526,7 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - pNode->datum.p = calloc(1, pNode->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pNode->datum.p) { code = TSDB_CODE_OUT_OF_MEMORY; break; @@ -1546,9 +1592,7 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - int32_t val; - code = tjsonGetIntValue(pJson, jkOperatorType, &val); - pNode->opType = val; + code = tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType); } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft); @@ -1582,9 +1626,7 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - int32_t val; - code = tjsonGetIntValue(pJson, jkLogicCondType, &val); - pNode->condType = val; + code = tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType); } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList); @@ -1807,6 +1849,84 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { return code; } +static const char* jkOrderByExprExpr = "Expr"; +static const char* jkOrderByExprOrder = "Order"; +static const char* jkOrderByExprNullOrder = "NullOrder"; + +static int32_t orderByExprNodeToJson(const void* pObj, SJson* pJson) { + const SOrderByExprNode* pNode = (const SOrderByExprNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkOrderByExprExpr, nodeToJson, pNode->pExpr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkOrderByExprOrder, pNode->order); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkOrderByExprNullOrder, pNode->nullOrder); + } + + return code; +} + +static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) { + SOrderByExprNode* pNode = (SOrderByExprNode*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder); + } + + return code; +} + +static const char* jkIntervalWindowInterval = "Interval"; +static const char* jkIntervalWindowOffset = "Offset"; +static const char* jkIntervalWindowSliding = "Sliding"; +static const char* jkIntervalWindowFill = "Fill"; +static const char* jkIntervalWindowTsPk = "TsPk"; + +static int32_t intervalWindowNodeToJson(const void* pObj, SJson* pJson) { + const SIntervalWindowNode* pNode = (const SIntervalWindowNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkIntervalWindowInterval, nodeToJson, pNode->pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowOffset, nodeToJson, pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowSliding, nodeToJson, pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowFill, nodeToJson, pNode->pFill); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowTsPk, nodeToJson, pNode->pCol); + } + + return code; +} + +static int32_t jsonToIntervalWindowNode(const SJson* pJson, void* pObj) { + SIntervalWindowNode* pNode = (SIntervalWindowNode*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkIntervalWindowInterval, &pNode->pInterval); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowOffset, &pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowSliding, &pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowFill, &pNode->pFill); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowTsPk, &pNode->pCol); + } + + return code; +} + static const char* jkNodeListDataType = "DataType"; static const char* jkNodeListNodeList = "NodeList"; @@ -1980,6 +2100,7 @@ static const char* jkSelectStmtHaving = "Having"; static const char* jkSelectStmtOrderBy = "OrderBy"; static const char* jkSelectStmtLimit = "Limit"; static const char* jkSelectStmtSlimit = "Slimit"; +static const char* jkSelectStmtStmtName = "StmtName"; static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { const SSelectStmt* pNode = (const SSelectStmt*)pObj; @@ -2015,6 +2136,9 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSelectStmtSlimit, nodeToJson, pNode->pSlimit); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSelectStmtStmtName, pNode->stmtName); + } return code; } @@ -2053,6 +2177,9 @@ static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkSelectStmtSlimit, &pNode->pSlimit); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSelectStmtStmtName, pNode->stmtName); + } return code; } @@ -2116,11 +2243,13 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_GROUPING_SET: return groupingSetNodeToJson(pObj, pJson); case QUERY_NODE_ORDER_BY_EXPR: + return orderByExprNodeToJson(pObj, pJson); case QUERY_NODE_LIMIT: case QUERY_NODE_STATE_WINDOW: case QUERY_NODE_SESSION_WINDOW: - case QUERY_NODE_INTERVAL_WINDOW: break; + case QUERY_NODE_INTERVAL_WINDOW: + return intervalWindowNodeToJson(pObj, pJson); case QUERY_NODE_NODE_LIST: return nodeListNodeToJson(pObj, pJson); case QUERY_NODE_FILL: @@ -2178,7 +2307,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return physiExchangeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: - break; + return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: @@ -2218,11 +2347,13 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { // break; // case QUERY_NODE_GROUPING_SET: // return jsonToGroupingSetNode(pJson, pObj); - // case QUERY_NODE_ORDER_BY_EXPR: + case QUERY_NODE_ORDER_BY_EXPR: + return jsonToOrderByExprNode(pJson, pObj); // case QUERY_NODE_LIMIT: // case QUERY_NODE_STATE_WINDOW: // case QUERY_NODE_SESSION_WINDOW: - // case QUERY_NODE_INTERVAL_WINDOW: + case QUERY_NODE_INTERVAL_WINDOW: + return jsonToIntervalWindowNode(pJson, pObj); case QUERY_NODE_NODE_LIST: return jsonToNodeListNode(pJson, pObj); // case QUERY_NODE_FILL: @@ -2266,6 +2397,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiAggNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return jsonToPhysiExchangeNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_SORT: + return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return jsonToPhysiIntervalNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: @@ -2307,9 +2440,7 @@ static int32_t nodeToJson(const void* pObj, SJson* pJson) { static int32_t jsonToNode(const SJson* pJson, void* pObj) { SNode* pNode = (SNode*)pObj; - int32_t val = 0; - int32_t code = tjsonGetIntValue(pJson, jkNodeType, &val); - pNode->type = val; + int32_t code = tjsonGetNumberValue(pJson, jkNodeType, pNode->type); if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, nodesNodeName(pNode->type), jsonToSpecificNode, pNode); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index ff71c3bd58..7eaa049946 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -99,6 +99,9 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker if (DEAL_RES_ERROR != res) { res = walkNode(pInterval->pFill, order, walker, pContext); } + if (DEAL_RES_ERROR != res) { + res = walkNode(pInterval->pCol, order, walker, pContext); + } break; } case QUERY_NODE_NODE_LIST: @@ -225,6 +228,9 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit if (DEAL_RES_ERROR != res) { res = rewriteNode(&(pInterval->pFill), order, rewriter, pContext); } + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pInterval->pCol), order, rewriter, pContext); + } break; } case QUERY_NODE_NODE_LIST: @@ -294,10 +300,10 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa case SQL_CLAUSE_GROUP_BY: nodesWalkNode(pSelect->pHaving, walker, pContext); case SQL_CLAUSE_HAVING: - nodesWalkList(pSelect->pProjectionList, walker, pContext); - case SQL_CLAUSE_SELECT: nodesWalkList(pSelect->pOrderByList, walker, pContext); case SQL_CLAUSE_ORDER_BY: + nodesWalkList(pSelect->pProjectionList, walker, pContext); + case SQL_CLAUSE_SELECT: default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index d529ee2b1a..035a2f1caa 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -22,7 +22,7 @@ #include "thash.h" static SNode* makeNode(ENodeType type, size_t size) { - SNode* p = calloc(1, size); + SNode* p = taosMemoryCalloc(1, size); if (NULL == p) { return NULL; } @@ -159,6 +159,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SExchangeLogicNode)); case QUERY_NODE_LOGIC_PLAN_WINDOW: return makeNode(type, sizeof(SWindowLogicNode)); + case QUERY_NODE_LOGIC_PLAN_SORT: + return makeNode(type, sizeof(SSortLogicNode)); case QUERY_NODE_LOGIC_SUBPLAN: return makeNode(type, sizeof(SLogicSubplan)); case QUERY_NODE_LOGIC_PLAN: @@ -182,7 +184,7 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return makeNode(type, sizeof(SExchangePhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SORT: - return makeNode(type, sizeof(SNode)); + return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: @@ -207,9 +209,9 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { case QUERY_NODE_VALUE: { SValueNode* pValue = (SValueNode*)*pNode; - tfree(pValue->literal); + taosMemoryFreeClear(pValue->literal); if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { - tfree(pValue->datum.p); + taosMemoryFreeClear(pValue->datum.p); } break; @@ -222,8 +224,8 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { break; case QUERY_NODE_REAL_TABLE: { SRealTableNode* pReal = (SRealTableNode*)*pNode; - tfree(pReal->pMeta); - tfree(pReal->pVgroupList); + taosMemoryFreeClear(pReal->pMeta); + taosMemoryFreeClear(pReal->pVgroupList); break; } case QUERY_NODE_TEMP_TABLE: @@ -262,8 +264,8 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { size_t size = taosArrayGetSize(pStmt->pDataBlocks); for (size_t i = 0; i < size; ++i) { SVgDataBlocks* pVg = taosArrayGetP(pStmt->pDataBlocks, i); - tfree(pVg->pData); - tfree(pVg); + taosMemoryFreeClear(pVg->pData); + taosMemoryFreeClear(pVg); } taosArrayDestroy(pStmt->pDataBlocks); break; @@ -292,7 +294,7 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { default: break; } - tfree(*pNode); + taosMemoryFreeClear(*pNode); return DEAL_RES_CONTINUE; } @@ -304,7 +306,7 @@ void nodesDestroyNode(SNodeptr pNode) { } SNodeList* nodesMakeList() { - SNodeList* p = calloc(1, sizeof(SNodeList)); + SNodeList* p = taosMemoryCalloc(1, sizeof(SNodeList)); if (NULL == p) { return NULL; } @@ -315,7 +317,7 @@ int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode) { if (NULL == pList || NULL == pNode) { return TSDB_CODE_SUCCESS; } - SListCell* p = calloc(1, sizeof(SListCell)); + SListCell* p = taosMemoryCalloc(1, sizeof(SListCell)); if (NULL == p) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; @@ -370,7 +372,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { } pTarget->pTail = pSrc->pTail; pTarget->length += pSrc->length; - tfree(pSrc); + taosMemoryFreeClear(pSrc); return TSDB_CODE_SUCCESS; } @@ -395,7 +397,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { } SListCell* pNext = pCell->pNext; nodesDestroyNode(pCell->pNode); - tfree(pCell); + taosMemoryFreeClear(pCell); --(pList->length); return pNext; } @@ -419,7 +421,7 @@ void nodesDestroyList(SNodeList* pList) { while (NULL != pNext) { pNext = nodesListErase(pList, pNext); } - tfree(pList); + taosMemoryFreeClear(pList); } void nodesClearList(SNodeList* pList) { @@ -431,9 +433,9 @@ void nodesClearList(SNodeList* pList) { while (NULL != pNext) { SListCell* tmp = pNext; pNext = pNext->pNext; - tfree(tmp); + taosMemoryFreeClear(tmp); } - tfree(pList); + taosMemoryFreeClear(pList); } void* nodesGetValueFromNode(SValueNode *pNode) { @@ -555,7 +557,7 @@ static EDealRes collectColumns(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; int32_t colId = pCol->colId; - if (0 == strcmp(pCxt->pTableAlias, pCol->tableAlias)) { + if (NULL == pCxt->pTableAlias || 0 == strcmp(pCxt->pTableAlias, pCol->tableAlias)) { return doCollect(pCxt, colId, pNode); } } diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index acd021572d..a38d64c58c 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -41,26 +41,26 @@ typedef struct SBoundColumn { } SBoundColumn; typedef struct { - uint16_t schemaColIdx; - uint16_t boundIdx; - uint16_t finalIdx; + col_id_t schemaColIdx; + col_id_t boundIdx; + col_id_t finalIdx; } SBoundIdxInfo; typedef struct SParsedDataColInfo { - int16_t numOfCols; - int16_t numOfBound; + col_id_t numOfCols; + col_id_t numOfBound; uint16_t flen; // TODO: get from STSchema uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) uint16_t extendedVarLen; uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - int32_t * boundedColumns; // bound column idx according to schema - SBoundColumn * cols; + col_id_t *boundColumns; // bound column idx according to schema + SBoundColumn *cols; SBoundIdxInfo *colIdxInfo; int8_t orderStatus; // bound columns } SParsedDataColInfo; typedef struct { - uint8_t memRowType; // default is 0, that is SDataRow + uint8_t rowType; // default is 0, that is SDataRow int32_t rowSize; } SMemRowBuilder; @@ -92,11 +92,11 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1); } -static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, - int32_t idx, int32_t *toffset, int32_t *colIdx) { - int32_t schemaIdx = 0; +static FORCE_INLINE void getSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, + int32_t *toffset, col_id_t *colIdx) { + col_id_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundedColumns[idx] - PRIMARYKEY_TIMESTAMP_COL_ID; + schemaIdx = spd->boundColumns[idx] - PRIMARYKEY_TIMESTAMP_COL_ID; if (TD_IS_TP_ROW_T(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart *colIdx = schemaIdx; @@ -132,7 +132,7 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); -void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); +void setBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); void destroyBoundColumnInfo(SParsedDataColInfo* pColList); void destroyBlockArrayList(SArray* pDataBlockList); void destroyBlockHashmap(SHashObj* pDataBlockHash); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 171b406e18..742ab303d3 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -30,6 +30,8 @@ extern "C" { #define parserDebug(param, ...) qDebug("PARSER: " param, __VA_ARGS__) #define parserTrace(param, ...) qTrace("PARSER: " param, __VA_ARGS__) +#define PK_TS_COL_INTERNAL_NAME "_rowts" + typedef struct SMsgBuf { int32_t len; char *buf; diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 4f27743b2b..a958a748e2 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -481,7 +481,7 @@ SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { CHECK_RAW_EXPR_NODE(pNode); SNode* tmp = ((SRawExprNode*)pNode)->pNode; - tfree(pNode); + taosMemoryFreeClear(pNode); return tmp; } @@ -645,6 +645,11 @@ SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const STok tempTable->pSubquery = pSubquery; if (NULL != pTableAlias && TK_NK_NIL != pTableAlias->type) { strncpy(tempTable->table.tableAlias, pTableAlias->z, pTableAlias->n); + } else { + sprintf(tempTable->table.tableAlias, "%p", tempTable); + } + if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { + strcpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias); } return (SNode*)tempTable; } @@ -697,6 +702,13 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol) { SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { SIntervalWindowNode* interval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); CHECK_OUT_OF_MEM(interval); + interval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == interval->pCol) { + nodesDestroyNode(interval); + CHECK_OUT_OF_MEM(interval->pCol); + } + ((SColumnNode*)interval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + strcpy(((SColumnNode*)interval->pCol)->colName, PK_TS_COL_INTERNAL_NAME); interval->pInterval = pInterval; interval->pOffset = pOffset; interval->pSliding = pSliding; @@ -792,6 +804,7 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr select->isDistinct = isDistinct; select->pProjectionList = pProjectionList; select->pFromTable = pTable; + sprintf(select->stmtName, "%p", select); return (SNode*)select; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index e992f150aa..5c38ccaff8 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -29,7 +29,7 @@ extern void ParseTrace(FILE*, char*); int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); - void *pParser = ParseAlloc(malloc); + void *pParser = ParseAlloc((FMalloc)taosMemoryMalloc); int32_t i = 0; while (1) { SToken t0 = {0}; @@ -73,9 +73,9 @@ int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { } abort_parse: - ParseFree(pParser, free); + ParseFree(pParser, (FFree)taosMemoryFree); if (cxt.valid) { - *pQuery = calloc(1, sizeof(SQuery)); + *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 6e01b7b12e..ed67de17e0 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -277,7 +277,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { } for (size_t i = 0; i < numOfVg; ++i) { STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i); - SVgDataBlocks* dst = calloc(1, sizeof(SVgDataBlocks)); + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -600,9 +600,9 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int typedef struct SMemParam { SRowBuilder* rb; - SSchema* schema; - int32_t toffset; - int32_t colIdx; + SSchema* schema; + int32_t toffset; + col_id_t colIdx; } SMemParam; static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* param) { @@ -623,9 +623,11 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); } else { if (value == NULL) { // it is a null data - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, + pa->colIdx); } else { - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, + pa->colIdx); } } return TSDB_CODE_SUCCESS; @@ -633,18 +635,18 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p // pSql -> tag1_name, ...) static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { - int32_t nCols = pColList->numOfCols; + col_id_t nCols = pColList->numOfCols; pColList->numOfBound = 0; pColList->boundNullLen = 0; - memset(pColList->boundedColumns, 0, sizeof(int32_t) * nCols); - for (int32_t i = 0; i < nCols; ++i) { + memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); + for (col_id_t i = 0; i < nCols; ++i) { pColList->cols[i].valStat = VAL_STAT_NONE; } SToken sToken; bool isOrdered = true; - int32_t lastColIdx = -1; // last column found + col_id_t lastColIdx = -1; // last column found while (1) { NEXT_TOKEN(pCxt->pSql, sToken); @@ -652,8 +654,8 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* break; } - int32_t t = lastColIdx + 1; - int32_t index = findCol(&sToken, t, nCols, pSchema); + col_id_t t = lastColIdx + 1; + col_id_t index = findCol(&sToken, t, nCols, pSchema); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; @@ -666,7 +668,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* } lastColIdx = index; pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundedColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; + pColList->boundColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; ++pColList->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: @@ -684,23 +686,24 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; if (!isOrdered) { - pColList->colIdxInfo = calloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); + pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); if (NULL == pColList->colIdxInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (uint16_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = (uint16_t)pColList->boundedColumns[i]; + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].schemaColIdx = pColList->boundColumns[i]; pColIdx[i].boundIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); - for (uint16_t i = 0; i < pColList->numOfBound; ++i) { + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { pColIdx[i].finalIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - memset(&pColList->boundedColumns[pColList->numOfBound], 0, sizeof(int32_t) * (pColList->numOfCols - pColList->numOfBound)); + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); return TSDB_CODE_SUCCESS; } @@ -714,8 +717,8 @@ typedef struct SKvParam { static int32_t KvRowAppend(const void *value, int32_t len, void *param) { SKvParam* pa = (SKvParam*) param; - int32_t type = pa->schema->type; - int32_t colId = pa->schema->colId; + int8_t type = pa->schema->type; + int16_t colId = pa->schema->colId; if (TSDB_DATA_TYPE_BINARY == type) { STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); @@ -747,7 +750,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]]; + SSchema* pSchema = &pTagsSchema[pCxt->tags.boundColumns[i]]; param.schema = pSchema; CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); } @@ -760,7 +763,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, // todo construct payload - tfree(row); + taosMemoryFreeClear(row); return 0; } @@ -813,9 +816,9 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema *pSchema = &schema[spd->boundedColumns[i] - 1]; + SSchema* pSchema = &schema[spd->boundColumns[i] - 1]; param.schema = pSchema; - getMemRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); + getSTSRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, ¶m, &pCxt->msg)); if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { @@ -903,7 +906,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da } static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { - tfree(pCxt->pTableMeta); + taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); tdDestroyKVRowBuilder(&pCxt->tagsBuilder); } @@ -913,16 +916,16 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { return; } - tfree(pDataBlock->pData); + taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - tfree(pDataBlock->pTableMeta); + taosMemoryFreeClear(pDataBlock->pTableMeta); } destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } - tfree(pDataBlock); + taosMemoryFreeClear(pDataBlock); } static void destroyInsertParseContext(SInsertParseContext* pCxt) { @@ -1029,7 +1032,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - *pQuery = calloc(1, sizeof(SQuery)); + *pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 0189ddb5ad..f70e514b5a 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -43,12 +43,12 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { } } -void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols) { +void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { pColList->numOfCols = numOfCols; pColList->numOfBound = numOfCols; pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColList->boundedColumns = calloc(pColList->numOfCols, sizeof(int32_t)); - pColList->cols = calloc(pColList->numOfCols, sizeof(SBoundColumn)); + pColList->boundColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(col_id_t)); + pColList->cols = taosMemoryCalloc(pColList->numOfCols, sizeof(SBoundColumn)); pColList->colIdxInfo = NULL; pColList->flen = 0; pColList->allNullLen = 0; @@ -73,7 +73,7 @@ void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t default: break; } - pColList->boundedColumns[i] = pSchema[i].colId; + pColList->boundColumns[i] = pSchema[i].colId; } pColList->allNullLen += pColList->flen; pColList->boundNullLen = pColList->allNullLen; // default set allNullLen @@ -103,14 +103,14 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) { } void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { - tfree(pColList->boundedColumns); - tfree(pColList->cols); - tfree(pColList->colIdxInfo); + taosMemoryFreeClear(pColList->boundColumns); + taosMemoryFreeClear(pColList->cols); + taosMemoryFreeClear(pColList->colIdxInfo); } static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { - STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); + STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -123,9 +123,9 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->nAllocSize = dataBuf->headerSize * 2; } - dataBuf->pData = malloc(dataBuf->nAllocSize); + dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { - tfree(dataBuf); + taosMemoryFreeClear(dataBuf); return TSDB_CODE_TSC_OUT_OF_MEMORY; } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); @@ -190,16 +190,16 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { return; } - tfree(pDataBlock->pData); + taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - tfree(pDataBlock->pTableMeta); + taosMemoryFreeClear(pDataBlock->pTableMeta); } destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } - tfree(pDataBlock); + taosMemoryFreeClear(pDataBlock); } void destroyBlockArrayList(SArray* pDataBlockList) { @@ -283,7 +283,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey // allocate memory size_t nAlloc = nRows * sizeof(SBlockKeyTuple); if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char *tmp = realloc(pBlkKeyInfo->pKeyTuple, nAlloc); + char *tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -433,7 +433,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (ret != TSDB_CODE_SUCCESS) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return ret; } @@ -444,14 +444,14 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (dataBuf->nAllocSize < destSize) { dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - char* tmp = realloc(dataBuf->pData, dataBuf->nAllocSize); + char* tmp = taosMemoryRealloc(dataBuf->pData, dataBuf->nAllocSize); if (tmp != NULL) { dataBuf->pData = tmp; } else { // failed to allocate memory, free already allocated memory and return error code taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(dataBuf->pData); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return TSDB_CODE_TSC_OUT_OF_MEMORY; } } @@ -462,8 +462,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if ((code = sortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(dataBuf->pData); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); return code; } ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); @@ -492,7 +492,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t // free the table data blocks; taosHashCleanup(pVnodeDataBlockHashList); - tfree(blkKeyInfo.pKeyTuple); + taosMemoryFreeClear(blkKeyInfo.pKeyTuple); *pVgDataBlocks = pVnodeDataBlockList; return TSDB_CODE_SUCCESS; } @@ -509,7 +509,7 @@ int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t remain = pDataBlock->nAllocSize - pDataBlock->size; } - char *tmp = realloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); + char *tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); if (tmp != NULL) { pDataBlock->pData = tmp; memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 1dc1945f23..79651cd325 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -594,7 +594,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) { int32_t bsize = (int32_t)((uint64_t)token->z - (uint64_t)src); SToken ntoken; - *str = calloc(1, size); + *str = taosMemoryCalloc(1, size); strncpy(*str, src, bsize); strcat(*str, newToken); @@ -603,7 +603,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) { ntoken.n = (uint32_t)nsize; ntoken.z = *str + bsize; - tfree(src); + taosMemoryFreeClear(src); return ntoken; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8c842db161..60ee8be76d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -84,12 +84,18 @@ static SName* toName(int32_t acctId, const char* pDbName, const char* pTableName return pName; } -static int32_t collectUseDatabase(const char* pFullDbName, SHashObj* pDbs) { +static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) { SFullDatabaseName name = {0}; strcpy(name.fullDbName, pFullDbName); return taosHashPut(pDbs, pFullDbName, strlen(pFullDbName), &name, sizeof(SFullDatabaseName)); } +static int32_t collectUseDatabase(const SName* pName, SHashObj* pDbs) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pName, dbFName); + return collectUseDatabaseImpl(dbFName, pDbs); +} + static int32_t collectUseTable(const SName* pName, SHashObj* pDbs) { char fullName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pName, fullName); @@ -98,7 +104,10 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pDbs) { static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STableMeta** pMeta) { SParseContext* pParCxt = pCxt->pParseCxt; - int32_t code = collectUseTable(pName, pCxt->pTables); + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } if (TSDB_CODE_SUCCESS == code) { code = catalogGetTableMeta(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pMeta); } @@ -117,7 +126,10 @@ static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const static int32_t getTableDistVgInfo(STranslateContext* pCxt, const SName* pName, SArray** pVgInfo) { SParseContext* pParCxt = pCxt->pParseCxt; - int32_t code = collectUseTable(pName, pCxt->pTables); + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } if (TSDB_CODE_SUCCESS == code) { code = catalogGetTableDistVgInfo(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pVgInfo); } @@ -131,7 +143,7 @@ static int32_t getDBVgInfoImpl(STranslateContext* pCxt, const SName* pName, SArr SParseContext* pParCxt = pCxt->pParseCxt; char fullDbName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pName, fullDbName); - int32_t code = collectUseDatabase(fullDbName, pCxt->pDbs); + int32_t code = collectUseDatabaseImpl(fullDbName, pCxt->pDbs); if (TSDB_CODE_SUCCESS == code) { code = catalogGetDBVgInfo(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, fullDbName, pVgInfo); } @@ -151,7 +163,10 @@ static int32_t getDBVgInfo(STranslateContext* pCxt, const char* pDbName, SArray* static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pName, SVgroupInfo* pInfo) { SParseContext* pParCxt = pCxt->pParseCxt; - int32_t code = collectUseTable(pName, pCxt->pTables); + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } if (TSDB_CODE_SUCCESS == code) { code = catalogGetTableHashVgroup(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pInfo); } @@ -170,7 +185,7 @@ static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, int32_t* pTableNum) { SParseContext* pParCxt = pCxt->pParseCxt; - int32_t code = collectUseDatabase(pDbFName, pCxt->pDbs); + int32_t code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs); if (TSDB_CODE_SUCCESS == code) { code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum); } @@ -256,6 +271,10 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { bool found = false; if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; + if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && 0 == strcmp(pCol->colName, PK_TS_COL_INTERNAL_NAME)) { + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema, false, pCol); + return true; + } int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; for (int32_t i = 0; i < nums; ++i) { if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { @@ -383,12 +402,12 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - pVal->datum.p = calloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + pVal->datum.p = taosMemoryCalloc(1, pVal->node.resType.bytes + VARSTR_HEADER_SIZE + 1); if (NULL == pVal->datum.p) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); } varDataSetLen(pVal->datum.p, pVal->node.resType.bytes); - strcpy(varDataVal(pVal->datum.p), pVal->literal); + strncpy(varDataVal(pVal->datum.p), pVal->literal, pVal->node.resType.bytes); break; } case TSDB_DATA_TYPE_TIMESTAMP: { @@ -585,7 +604,7 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) static int32_t toVgroupsInfo(SArray* pVgs, SVgroupsInfo** pVgsInfo) { size_t vgroupNum = taosArrayGetSize(pVgs); - *pVgsInfo = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum); + *pVgsInfo = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum); if (NULL == *pVgsInfo) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -599,9 +618,9 @@ static int32_t toVgroupsInfo(SArray* pVgs, SVgroupsInfo** pVgsInfo) { static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { // todo release - // if (0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { - // return TSDB_CODE_SUCCESS; - // } + if (0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { + return TSDB_CODE_SUCCESS; + } int32_t code = TSDB_CODE_SUCCESS; SArray* vgroupList = NULL; @@ -613,9 +632,9 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea if (TSDB_CODE_SUCCESS == code) { // todo remove - if (NULL != vgroupList && taosArrayGetSize(vgroupList) > 0 && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { - taosArrayPopTailBatch(vgroupList, taosArrayGetSize(vgroupList) - 1); - } + //if (NULL != vgroupList && taosArrayGetSize(vgroupList) > 0 && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { + // taosArrayPopTailBatch(vgroupList, taosArrayGetSize(vgroupList) - 1); + //} code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); } @@ -640,7 +659,7 @@ static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTa } else if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { code = setSysTableVgroupList(pCxt, pName, pRealTable); } else { - pRealTable->pVgroupList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); if (NULL == pRealTable->pVgroupList) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -932,14 +951,14 @@ static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseS SCreateDbReq createReq = {0}; buildCreateDbReq(pCxt, pStmt, &createReq); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DB; pCxt->pCmdMsg->msgLen = tSerializeSCreateDbReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -955,14 +974,14 @@ static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt* tNameGetFullDbName(&name, dropReq.db); dropReq.ignoreNotExists = pStmt->ignoreNotExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_DB; pCxt->pCmdMsg->msgLen = tSerializeSDropDbReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -990,14 +1009,14 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm SAlterDbReq alterReq = {0}; buildAlterDbReq(pCxt, pStmt, &alterReq); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_DB; pCxt->pCmdMsg->msgLen = tSerializeSAlterDbReq(NULL, 0, &alterReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1041,7 +1060,7 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt strcpy(tableName.tname, pStmt->tableName); tNameExtractFullName(&tableName, createReq.name); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { tFreeSMCreateStbReq(&createReq); return TSDB_CODE_OUT_OF_MEMORY; @@ -1049,7 +1068,7 @@ static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableSt pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_STB; pCxt->pCmdMsg->msgLen = tSerializeSMCreateStbReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { tFreeSMCreateStbReq(&createReq); return TSDB_CODE_OUT_OF_MEMORY; @@ -1065,14 +1084,14 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p tNameExtractFullName(pTableName, dropReq.name); dropReq.igNotExists = ignoreNotExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_STB; pCxt->pCmdMsg->msgLen = tSerializeSMDropStbReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1095,7 +1114,7 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt // todo : drop normal table or child table code = TSDB_CODE_FAILED; } - tfree(pTableMeta); + taosMemoryFreeClear(pTableMeta); } return code; @@ -1159,14 +1178,14 @@ static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pSt } } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_STB; pCxt->pCmdMsg->msgLen = tSerializeSMAlterStbReq(NULL, 0, &alterReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1185,14 +1204,14 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p return code; } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_USE_DB; pCxt->pCmdMsg->msgLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1208,14 +1227,14 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt createReq.superUser = 0; strcpy(createReq.pass, pStmt->password); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_USER; pCxt->pCmdMsg->msgLen = tSerializeSCreateUserReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1234,14 +1253,14 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt strcpy(alterReq.dbname, pCxt->pParseCxt->db); } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_ALTER_USER; pCxt->pCmdMsg->msgLen = tSerializeSAlterUserReq(NULL, 0, &alterReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1254,14 +1273,14 @@ static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) SDropUserReq dropReq = {0}; strcpy(dropReq.user, pStmt->useName); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_USER; pCxt->pCmdMsg->msgLen = tSerializeSDropUserReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1275,14 +1294,14 @@ static int32_t translateCreateDnode(STranslateContext* pCxt, SCreateDnodeStmt* p strcpy(createReq.fqdn, pStmt->fqdn); createReq.port = pStmt->port; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1297,14 +1316,14 @@ static int32_t translateDropDnode(STranslateContext* pCxt, SDropDnodeStmt* pStmt strcpy(dropReq.fqdn, pStmt->fqdn); dropReq.port = pStmt->port; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSDropDnodeReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1319,14 +1338,14 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt strcpy(cfgReq.config, pStmt->config); strcpy(cfgReq.value, pStmt->value); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CONFIG_DNODE; pCxt->pCmdMsg->msgLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1366,14 +1385,14 @@ static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) { // tNameGetFullDbName(&name, showReq.db); // } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_SHOW; pCxt->pCmdMsg->msgLen = tSerializeSShowReq(NULL, 0, &showReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1383,7 +1402,7 @@ static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) { } static int32_t translateShowTables(STranslateContext* pCxt) { - SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + SVShowTablesReq* pShowReq = taosMemoryCalloc(1, sizeof(SVShowTablesReq)); SArray* array = NULL; int32_t code = getDBVgInfo(pCxt, pCxt->pParseCxt->db, &array); @@ -1393,7 +1412,7 @@ static int32_t translateShowTables(STranslateContext* pCxt) { SVgroupInfo* info = taosArrayGet(array, 0); pShowReq->head.vgId = htonl(info->vgId); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1433,6 +1452,7 @@ static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pS if (NULL == pSelect) { return TSDB_CODE_OUT_OF_MEMORY; } + sprintf(pSelect->stmtName, "%p", pSelect); SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); if (NULL == pTable) { @@ -1448,6 +1468,10 @@ static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pS nodesDestroyNode(pSelect); return TSDB_CODE_OUT_OF_MEMORY; } + SNode* pProject = NULL; + FOREACH(pProject, pSelect->pProjectionList) { + sprintf(((SExprNode*)pProject)->aliasName, "#sma_%p", pProject); + } SIntervalWindowNode* pInterval = nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); if (NULL == pInterval) { @@ -1455,14 +1479,18 @@ static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pS return TSDB_CODE_OUT_OF_MEMORY; } pSelect->pWindow = (SNode*)pInterval; + pInterval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); pInterval->pInterval = nodesCloneNode(pStmt->pOptions->pInterval); pInterval->pOffset = nodesCloneNode(pStmt->pOptions->pOffset); pInterval->pSliding = nodesCloneNode(pStmt->pOptions->pSliding); - if (NULL == pInterval->pInterval || (NULL != pStmt->pOptions->pOffset && NULL == pInterval->pOffset) || + if (NULL == pInterval->pCol || NULL == pInterval->pInterval || + (NULL != pStmt->pOptions->pOffset && NULL == pInterval->pOffset) || (NULL != pStmt->pOptions->pSliding && NULL == pInterval->pSliding)) { nodesDestroyNode(pSelect); return TSDB_CODE_OUT_OF_MEMORY; } + ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + strcpy(((SColumnNode*)pInterval->pCol)->colName, PK_TS_COL_INTERNAL_NAME); int32_t code = translateQuery(pCxt, (SNode*)pSelect); if (TSDB_CODE_SUCCESS == code) { @@ -1514,14 +1542,14 @@ static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt return code; } - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_SMA; pCxt->pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, &createSmaReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1544,14 +1572,14 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt SVDropTSmaReq dropSmaReq = {0}; strcpy(dropSmaReq.indexName, pStmt->indexName); - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA; pCxt->pCmdMsg->msgLen = tSerializeSVDropTSmaReq(NULL, &dropSmaReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1564,14 +1592,14 @@ static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* pStmt) { SMCreateQnodeReq createReq = { .dnodeId = pStmt->dnodeId }; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_DND_CREATE_QNODE; pCxt->pCmdMsg->msgLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1583,14 +1611,14 @@ static int32_t translateCreateQnode(STranslateContext* pCxt, SCreateQnodeStmt* p static int32_t translateDropQnode(STranslateContext* pCxt, SDropQnodeStmt* pStmt) { SDDropQnodeReq dropReq = { .dnodeId = pStmt->dnodeId }; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_DND_DROP_QNODE; pCxt->pCmdMsg->msgLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1626,14 +1654,14 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p tNameExtractFullName(&name, createReq.name); createReq.igExists = pStmt->ignoreExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_CREATE_TOPIC; pCxt->pCmdMsg->msgLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1652,14 +1680,14 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt tNameExtractFullName(&name, dropReq.name); dropReq.igNotExists = pStmt->ignoreNotExists; - pCxt->pCmdMsg = malloc(sizeof(SCmdMsgInfo)); + pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); if (NULL == pCxt->pCmdMsg) { return TSDB_CODE_OUT_OF_MEMORY; } pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; pCxt->pCmdMsg->msgType = TDMT_MND_DROP_TOPIC; pCxt->pCmdMsg->msgLen = tSerializeSMDropTopicReq(NULL, 0, &dropReq); - pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen); + pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); if (NULL == pCxt->pCmdMsg->pMsg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1772,10 +1800,10 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { } int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + if (NULL != pRoot && QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { SSelectStmt* pSelect = (SSelectStmt*) pRoot; *numOfCols = LIST_LENGTH(pSelect->pProjectionList); - *pSchema = calloc((*numOfCols), sizeof(SSchema)); + *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1805,8 +1833,8 @@ static void destroyTranslateContext(STranslateContext* pCxt) { } if (NULL != pCxt->pCmdMsg) { - tfree(pCxt->pCmdMsg->pMsg); - tfree(pCxt->pCmdMsg); + taosMemoryFreeClear(pCxt->pCmdMsg->pMsg); + taosMemoryFreeClear(pCxt->pCmdMsg); } taosHashCleanup(pCxt->pDbs); @@ -1850,6 +1878,7 @@ static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) if (NULL == pSelect) { return TSDB_CODE_OUT_OF_MEMORY; } + sprintf(pSelect->stmtName, "%p", pSelect); SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); if (NULL == pTable) { @@ -1858,6 +1887,7 @@ static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) } strcpy(pTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB); strcpy(pTable->table.tableName, getSysTableName(showType)); + strcpy(pTable->table.tableAlias, pTable->table.tableName); pSelect->pFromTable = (SNode*)pTable; *pStmt = pSelect; @@ -1947,6 +1977,7 @@ static int32_t rewriteShow(STranslateContext* pCxt, SQuery* pQuery) { code = createShowCondition((SShowStmt*)pQuery->pRoot, pStmt); } if (TSDB_CODE_SUCCESS == code) { + pQuery->showRewrite = true; nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; } @@ -1959,7 +1990,7 @@ typedef struct SVgroupTablesBatch { char dbName[TSDB_DB_NAME_LEN]; } SVgroupTablesBatch; -static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema) { +static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { pSchema->colId = colId; pSchema->type = pCol->dataType.type; pSchema->bytes = pCol->dataType.bytes; @@ -1967,9 +1998,9 @@ static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema } static void destroyCreateTbReq(SVCreateTbReq* pReq) { - tfree(pReq->dbFName); - tfree(pReq->name); - tfree(pReq->ntbCfg.pSchema); + taosMemoryFreeClear(pReq->dbFName); + taosMemoryFreeClear(pReq->name); + taosMemoryFreeClear(pReq->ntbCfg.pSchema); } static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, const char* pTableName, @@ -1984,7 +2015,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, con req.dbFName = strdup(dbFName); req.name = strdup(pTableName); req.ntbCfg.nCols = LIST_LENGTH(pColumns); - req.ntbCfg.pSchema = calloc(req.ntbCfg.nCols, sizeof(SSchema)); + req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema)); if (NULL == req.name || NULL == req.ntbCfg.pSchema) { destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; @@ -2010,7 +2041,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const char* pDbName, con static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) { int tlen = sizeof(SMsgHead) + tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req)); - void* buf = malloc(tlen); + void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2019,7 +2050,7 @@ static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tSerializeSVCreateTbBatchReq(&pBuf, &(pTbBatch->req)); - SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks)); + SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == pVgData) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2036,13 +2067,13 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { size_t size = taosArrayGetSize(pTbBatch->req.pArray); for(int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); - tfree(pTableReq->dbFName); - tfree(pTableReq->name); + taosMemoryFreeClear(pTableReq->dbFName); + taosMemoryFreeClear(pTableReq->name); if (pTableReq->type == TSDB_NORMAL_TABLE) { - tfree(pTableReq->ntbCfg.pSchema); + taosMemoryFreeClear(pTableReq->ntbCfg.pSchema); } else if (pTableReq->type == TSDB_CHILD_TABLE) { - tfree(pTableReq->ctbCfg.pTag); + taosMemoryFreeClear(pTableReq->ctbCfg.pTag); } } @@ -2065,8 +2096,8 @@ static void destroyCreateTbReqArray(SArray* pArray) { size_t size = taosArrayGetSize(pArray); for (size_t i = 0; i < size; ++i) { SVgDataBlocks* pVg = taosArrayGetP(pArray, i); - tfree(pVg->pData); - tfree(pVg); + taosMemoryFreeClear(pVg->pData); + taosMemoryFreeClear(pVg); } taosArrayDestroy(pArray); } @@ -2274,7 +2305,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt->dbName, pStmt->tableName, row, pSuperTableMeta->uid, &info); } - tfree(pSuperTableMeta); + taosMemoryFreeClear(pSuperTableMeta); tdDestroyKVRowBuilder(&kvRowBuilder); return code; } @@ -2369,13 +2400,14 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { } static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { - int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pQuery->pRoot)) { case QUERY_NODE_SELECT_STMT: pQuery->haveResultSet = true; pQuery->directRpc = false; pQuery->msgType = TDMT_VND_QUERY; - code = qExtractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema); + if (TSDB_CODE_SUCCESS != qExtractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema)) { + return TSDB_CODE_OUT_OF_MEMORY; + } break; case QUERY_NODE_VNODE_MODIF_STMT: pQuery->haveResultSet = false; @@ -2415,7 +2447,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { } } - return code; + return TSDB_CODE_SUCCESS; } int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 2e3936a6fb..80d04c5ee4 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -122,7 +122,7 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta) { assert(pTableMeta != NULL); size_t size = getTableMetaSize(pTableMeta); - STableMeta* p = malloc(size); + STableMeta* p = taosMemoryMalloc(size); memcpy(p, pTableMeta, size); return p; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 08f316b539..92bd6f111b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -53,10 +53,10 @@ void qDestroyQuery(SQuery* pQueryNode) { return; } nodesDestroyNode(pQueryNode->pRoot); - tfree(pQueryNode->pResSchema); + taosMemoryFreeClear(pQueryNode->pResSchema); if (NULL != pQueryNode->pCmdMsg) { - tfree(pQueryNode->pCmdMsg->pMsg); - tfree(pQueryNode->pCmdMsg); + taosMemoryFreeClear(pQueryNode->pCmdMsg->pMsg); + taosMemoryFreeClear(pQueryNode->pCmdMsg); } - tfree(pQueryNode); + taosMemoryFreeClear(pQueryNode); } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 8f9443d4ec..451587c8bf 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -73,7 +73,7 @@ ** which is ParseTOKENTYPE. The entry in the union ** for terminal symbols is called "yy0". ** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() +** zero the stack is dynamically sized using taosMemoryRealloc() ** ParseARG_SDECL A static variable declaration for the %extra_argument ** ParseARG_PDECL A parameter declaration for the %extra_argument ** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter @@ -1325,10 +1325,10 @@ static int yyGrowStack(yyParser *p){ newSize = p->yystksz*2 + 100; idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); + pNew = taosMemoryMalloc(newSize*sizeof(pNew[0])); if( pNew ) pNew[0] = p->yystk0; }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + pNew = taosMemoryRealloc(p->yystack, newSize*sizeof(pNew[0])); } if( pNew ){ p->yystack = pNew; @@ -1610,7 +1610,7 @@ void ParseFinalize(void *p){ yyParser *pParser = (yyParser*)p; while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); + if( pParser->yystack!=&pParser->yystk0 ) taosMemoryFree(pParser->yystack); #endif } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 4d80c47ec6..402caeb252 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -65,7 +65,7 @@ private: friend class MockCatalogServiceImpl; static std::unique_ptr createTableBuilder(int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { - STableMeta* meta = (STableMeta*)std::calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags)); + STableMeta* meta = (STableMeta*)taosMemoryCalloc(1, sizeof(STableMeta) + sizeof(SSchema) * (numOfColumns + numOfTags)); if (nullptr == meta) { throw std::bad_alloc(); } @@ -87,7 +87,7 @@ private: return meta_; } - int32_t colId_; + col_id_t colId_; int32_t rowsize_; std::shared_ptr meta_; }; @@ -277,7 +277,7 @@ private: return TSDB_CODE_TSC_INVALID_TABLE_NAME; } int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); - dst->reset((STableMeta*)std::calloc(1, len)); + dst->reset((STableMeta*)taosMemoryCalloc(1, len)); if (!dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index a722f0b821..887979f11e 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -44,7 +44,7 @@ public: struct MockTableMeta { ~MockTableMeta() { - free(schema); + taosMemoryFree(schema); } STableMeta* schema; diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index 3ffdf4d9b9..1813ad0827 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -99,7 +99,7 @@ private: throw "nodesNodeToString failed!"; } string str(pStr); - tfree(pStr); + taosMemoryFreeClear(pStr); return str; } @@ -358,7 +358,6 @@ TEST_F(ParserTest, selectSemanticError) { ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION)); } - TEST_F(ParserTest, showUsers) { setDatabase("root", "test"); @@ -366,8 +365,6 @@ TEST_F(ParserTest, showUsers) { ASSERT_TRUE(run()); } - - TEST_F(ParserTest, createDnode) { setDatabase("root", "test"); diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 42449d63d6..144254b042 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -22,32 +22,6 @@ extern "C" { #include "planner.h" -#define CHECK_ALLOC(p, res) \ - do { \ - if (NULL == (p)) { \ - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ - return (res); \ - } \ - } while (0) - -#define CHECK_CODE(exec, res) \ - do { \ - int32_t code = (exec); \ - if (TSDB_CODE_SUCCESS != code) { \ - pCxt->errCode = code; \ - return (res); \ - } \ - } while (0) - -#define CHECK_CODE_EXT(exec) \ - do { \ - int32_t code = (exec); \ - if (TSDB_CODE_SUCCESS != code) { \ - pCxt->errCode = code; \ - return code; \ - } \ - } while (0) - #define planFatal(param, ...) qFatal("PLAN: " param, __VA_ARGS__) #define planError(param, ...) qError("PLAN: " param, __VA_ARGS__) #define planWarn(param, ...) qWarn("PLAN: " param, __VA_ARGS__) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 84fa52a070..740fb678fd 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -45,7 +45,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { } if (nodesEqualNode(pExpr, *pNode)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - CHECK_ALLOC(pCol, DEAL_RES_ERROR); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); @@ -65,17 +67,12 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -typedef struct SNameExprCxt { - int32_t rewriteId; -} SNameExprCxt; - static EDealRes doNameExpr(SNode* pNode, void* pContext) { switch (nodeType(pNode)) { case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: { - SNameExprCxt* pCxt = (SNameExprCxt*)pContext; - sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d", pCxt->rewriteId++); + sprintf(((SExprNode*)pNode)->aliasName, "#expr_%p", pNode); return DEAL_RES_IGNORE_CHILD; } default: @@ -86,9 +83,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { } static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - static int32_t rewriteId = 1; - SNameExprCxt nameCxt = { .rewriteId = rewriteId }; - nodesWalkList(pExprs, doNameExpr, &nameCxt); + nodesWalkList(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; @@ -161,6 +156,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->tableName.acctId = pCxt->pPlanCxt->acctId; strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.tname, pRealTable->table.tableName); + pScan->showRewrite = pCxt->pPlanCxt->showRewrite; // set columns to scan SNodeList* pCols = NULL; @@ -290,13 +286,14 @@ static int32_t createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSel return code; } -static SColumnNode* createColumnByExpr(SExprNode* pExpr) { +static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) { SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { return NULL; } pCol->node.resType = pExpr->resType; strcpy(pCol->colName, pExpr->aliasName); + strcpy(pCol->tableAlias, pStmtName); return pCol; } @@ -310,20 +307,22 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { switch (nodeType(pNode)) { case QUERY_NODE_COLUMN: { SNode* pCol = nodesCloneNode(pNode); - CHECK_ALLOC(pCol, DEAL_RES_ERROR); - CHECK_CODE(nodesListAppend(pCxt->pList, pCol), DEAL_RES_ERROR); - return DEAL_RES_IGNORE_CHILD; + if (NULL == pCol) { + return DEAL_RES_ERROR; + } + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: { SExprNode* pExpr = (SExprNode*)pNode; SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - CHECK_ALLOC(pCol, DEAL_RES_ERROR); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } pCol->node.resType = pExpr->resType; strcpy(pCol->colName, pExpr->aliasName); - CHECK_CODE(nodesListAppend(pCxt->pList, (SNode*)pCol), DEAL_RES_ERROR); - return DEAL_RES_IGNORE_CHILD; + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } default: break; @@ -456,6 +455,12 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); + pWindow->pTspk = nodesCloneNode(pInterval->pCol); + if (NULL == pWindow->pTspk) { + nodesDestroyNode(pWindow); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (NULL != pInterval->pFill) { pWindow->pFill = nodesCloneNode(pInterval->pFill); if (NULL == pWindow->pFill) { @@ -484,7 +489,42 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return TSDB_CODE_FAILED; } -static int32_t createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pCols) { +static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + if (NULL == pSelect->pOrderByList) { + return TSDB_CODE_SUCCESS; + } + + SSortLogicNode* pSort = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pCols = NULL; + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, &pCols); + if (TSDB_CODE_SUCCESS == code && NULL != pCols) { + pSort->node.pTargets = nodesCloneList(pCols); + if (NULL == pSort->node.pTargets) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + pSort->pSortKeys = nodesCloneList(pSelect->pOrderByList); + if (NULL == pSort->pSortKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pSort; + } else { + nodesDestroyNode(pSort); + } + + return code; +} + +static int32_t createColumnByProjections(SLogicPlanContext* pCxt, const char* pStmtName, SNodeList* pExprs, SNodeList** pCols) { SNodeList* pList = nodesMakeList(); if (NULL == pList) { return TSDB_CODE_OUT_OF_MEMORY; @@ -492,7 +532,7 @@ static int32_t createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pEx SNode* pNode; FOREACH(pNode, pExprs) { - if (TSDB_CODE_SUCCESS != nodesListAppend(pList, createColumnByExpr((SExprNode*)pNode))) { + if (TSDB_CODE_SUCCESS != nodesListAppend(pList, createColumnByExpr(pStmtName, (SExprNode*)pNode))) { nodesDestroyList(pList); return TSDB_CODE_OUT_OF_MEMORY; } @@ -514,9 +554,10 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel if (NULL == pProject->pProjections) { code = TSDB_CODE_OUT_OF_MEMORY; } + strcpy(pProject->stmtName, pSelect->stmtName); if (TSDB_CODE_SUCCESS == code) { - code = createColumnByProjections(pCxt,pSelect->pProjectionList, &pProject->node.pTargets); + code = createColumnByProjections(pCxt, pSelect->stmtName, pSelect->pProjectionList, &pProject->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -537,6 +578,9 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); } + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createSortLogicNode, &pRoot); + } if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createProjectLogicNode, &pRoot); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4077c57f2c..c95845f8c7 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -17,9 +17,14 @@ #include "functionMgt.h" +typedef struct SSlotIdInfo { + int16_t slotId; + bool set; +} SSlotIdInfo; + typedef struct SSlotIndex { int16_t dataBlockId; - int16_t slotId; + SArray* pSlotIdsInfo; // duplicate name slot } SSlotIndex; typedef struct SPhysiPlanContext { @@ -30,74 +35,197 @@ typedef struct SPhysiPlanContext { SArray* pExecNodeList; } SPhysiPlanContext; -static int32_t getSlotKey(SNode* pNode, char* pKey) { +static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { + if (QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode)) { + return getSlotKey(((SOrderByExprNode*)pNode)->pExpr, pStmtName, pKey); + } + if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; + if (NULL != pStmtName) { + return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); + } if ('\0' == pCol->tableAlias[0]) { return sprintf(pKey, "%s", pCol->colName); } return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName); } + + if (NULL != pStmtName) { + return sprintf(pKey, "%s.%s", pStmtName, ((SExprNode*)pNode)->aliasName); + } return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); } -static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) { +static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId, bool output) { SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC); - CHECK_ALLOC(pSlot, NULL); + if (NULL == pSlot) { + return NULL; + } pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = false; - pSlot->output = true; + pSlot->output = output; return (SNode*)pSlot; } -static SNode* createTarget(SNode* pNode, int16_t dataBlockId, int16_t slotId) { +static int32_t createTarget(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { STargetNode* pTarget = (STargetNode*)nodesMakeNode(QUERY_NODE_TARGET); if (NULL == pTarget) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } + pTarget->dataBlockId = dataBlockId; pTarget->slotId = slotId; pTarget->pExpr = pNode; - return (SNode*)pTarget; + + *pOutput = (SNode*)pTarget; + return TSDB_CODE_SUCCESS; } -static int32_t addDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - SHashObj* pHash = NULL; - if (NULL == pDataBlockDesc->pSlots) { - pDataBlockDesc->pSlots = nodesMakeList(); - CHECK_ALLOC(pDataBlockDesc->pSlots, TSDB_CODE_OUT_OF_MEMORY); - - pHash = taosHashInit(LIST_LENGTH(pList), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - CHECK_ALLOC(pHash, TSDB_CODE_OUT_OF_MEMORY); - if (NULL == taosArrayInsert(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId, &pHash)) { - taosHashCleanup(pHash); - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { - pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); +static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char* pName, int32_t len, SHashObj* pHash) { + SSlotIndex* pIndex = taosHashGet(pHash, pName, len); + if (NULL != pIndex) { + SSlotIdInfo info = { .slotId = slotId, .set = false }; + taosArrayPush(pIndex->pSlotIdsInfo, &info); + return TSDB_CODE_SUCCESS; } - - SNode* pNode = NULL; - int16_t slotId = taosHashGetSize(pHash); - FOREACH(pNode, pList) { - CHECK_CODE_EXT(nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId))); - SSlotIndex index = { .dataBlockId = pDataBlockDesc->dataBlockId, .slotId = slotId }; - char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(pNode, name); - CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY); - - SNode* pTarget = createTarget(pNode, pDataBlockDesc->dataBlockId, slotId); - CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); - REPLACE_NODE(pTarget); - - pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes; - ++slotId; + SSlotIndex index = { .dataBlockId = dataBlockId, .pSlotIdsInfo = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SSlotIdInfo)) }; + if (NULL == index.pSlotIdsInfo) { + return TSDB_CODE_OUT_OF_MEMORY; } + SSlotIdInfo info = { .slotId = slotId, .set = false }; + taosArrayPush(index.pSlotIdsInfo, &info); + return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex)); +} + +static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) { + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + int32_t len = getSlotKey(pNode, NULL, name); + return putSlotToHashImpl(dataBlockId, slotId, name, len, pHash); +} + +static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId, SHashObj** pDescHash) { + SHashObj* pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pHash) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (NULL == taosArrayInsert(pCxt->pLocationHelper, dataBlockId, &pHash)) { + taosHashCleanup(pHash); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pDescHash = pHash; return TSDB_CODE_SUCCESS; } +static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, SHashObj* pHash) { + pDataBlockDesc->pSlots = nodesMakeList(); + if (NULL == pDataBlockDesc->pSlots) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + int16_t slotId = 0; + SNode* pNode = NULL; + FOREACH(pNode, pList) { + code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId, true)); + if (TSDB_CODE_SUCCESS == code) { + code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash); + } + if (TSDB_CODE_SUCCESS == code) { + pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes; + ++slotId; + } else { + break; + } + } + return code; +} + +static int32_t createDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode** pDataBlockDesc) { + SDataBlockDescNode* pDesc = nodesMakeNode(QUERY_NODE_DATABLOCK_DESC); + if (NULL == pDesc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pDesc->dataBlockId = pCxt->nextDataBlockId++; + + SHashObj* pHash = NULL; + int32_t code = createDataBlockDescHash(pCxt, LIST_LENGTH(pList), pDesc->dataBlockId, &pHash); + if (TSDB_CODE_SUCCESS == code) { + code = buildDataBlockSlots(pCxt, pList, pDesc, pHash); + } + + if (TSDB_CODE_SUCCESS == code) { + *pDataBlockDesc = pDesc; + } else { + nodesDestroyNode(pDesc); + } + + return code; +} + +static int16_t getUnsetSlotId(const SArray* pSlotIdsInfo) { + int32_t size = taosArrayGetSize(pSlotIdsInfo); + for (int32_t i = 0; i < size; ++i) { + SSlotIdInfo* pInfo = taosArrayGet(pSlotIdsInfo, i); + if (!pInfo->set) { + pInfo->set = true; + return pInfo->slotId; + } + } + return ((SSlotIdInfo*)taosArrayGet(pSlotIdsInfo, 0))->slotId; +} + +static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, const char* pStmtName, bool output) { + int32_t code = TSDB_CODE_SUCCESS; + SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); + int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0; + SNode* pNode = NULL; + FOREACH(pNode, pList) { + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN] = {0}; + int32_t len = getSlotKey(pNode, pStmtName, name); + SSlotIndex* pIndex = taosHashGet(pHash, name, len); + if (NULL == pIndex) { + code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, nextSlotId, output)); + if (TSDB_CODE_SUCCESS == code) { + code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash); + } + pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes; + slotId = nextSlotId; + ++nextSlotId; + } else { + slotId = getUnsetSlotId(pIndex->pSlotIdsInfo); + } + + if (TSDB_CODE_SUCCESS == code) { + SNode* pTarget = NULL; + code = createTarget(pNode, pDataBlockDesc->dataBlockId, slotId, &pTarget); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(pTarget); + } + } + + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t addDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, false); +} + +static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true); +} + +static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, true); +} + typedef struct SSetSlotIdCxt { int32_t errCode; SHashObj* pLeftHash; @@ -108,16 +236,17 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) { SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(pNode, name); + int32_t len = getSlotKey(pNode, NULL, name); SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len); if (NULL == pIndex) { pIndex = taosHashGet(pCxt->pRightHash, name, len); } // pIndex is definitely not NULL, otherwise it is a bug - CHECK_ALLOC(pIndex, DEAL_RES_ERROR); + if (NULL == pIndex) { + return DEAL_RES_ERROR; + } ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; - ((SColumnNode*)pNode)->slotId = pIndex->slotId; - CHECK_ALLOC(pNode, DEAL_RES_ERROR); + ((SColumnNode*)pNode)->slotId = ((SSlotIdInfo*)taosArrayGet(pIndex->pSlotIdsInfo, 0))->slotId; return DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -144,7 +273,7 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i return TSDB_CODE_SUCCESS; } -static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNodeList* pList, SNodeList** pOutput) { +static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, const SNodeList* pList, SNodeList** pOutput) { SNodeList* pRes = nodesCloneList(pList); if (NULL == pRes) { return TSDB_CODE_OUT_OF_MEMORY; @@ -164,18 +293,17 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i return TSDB_CODE_SUCCESS; } -static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { +static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, ENodeType type) { SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); if (NULL == pPhysiNode) { return NULL; } - pPhysiNode->pOutputDataBlockDesc = nodesMakeNode(QUERY_NODE_DATABLOCK_DESC); - if (NULL == pPhysiNode->pOutputDataBlockDesc) { + + int32_t code = createDataBlockDesc(pCxt, pLogicNode->pTargets, &pPhysiNode->pOutputDataBlockDesc); + if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode(pPhysiNode); return NULL; } - pPhysiNode->pOutputDataBlockDesc->dataBlockId = pCxt->nextDataBlockId++; - pPhysiNode->pOutputDataBlockDesc->type = QUERY_NODE_DATABLOCK_DESC; return pPhysiNode; } @@ -186,24 +314,11 @@ static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pL return TSDB_CODE_SUCCESS; } -static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, SDataBlockDescNode* pDataBlockDesc) { - SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); - char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - SNode* pNode; - FOREACH(pNode, pTargets) { - int32_t len = getSlotKey(pNode, name); - SSlotIndex* pIndex = taosHashGet(pHash, name, len); - // pIndex is definitely not NULL, otherwise it is a bug - CHECK_ALLOC(pIndex, TSDB_CODE_FAILED); - ((SSlotDescNode*)nodesListGetNode(pDataBlockDesc->pSlots, pIndex->slotId))->output = true; - } - - return TSDB_CODE_SUCCESS; -} - static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) { SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); - CHECK_ALLOC(pCol, NULL); + if (NULL == pCol) { + return NULL; + } pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; pCol->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; pCol->tableId = tableId; @@ -213,12 +328,43 @@ static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) { return pCol; } +static int32_t colIdCompare(const void* pLeft, const void* pRight) { + SColumnNode* pLeftCol = *(SColumnNode**)pLeft; + SColumnNode* pRightCol = *(SColumnNode**)pRight; + return pLeftCol->colId > pRightCol->colId ? 1 : -1; +} + +static int32_t sortScanCols(SNodeList* pScanCols) { + SArray* pArray = taosArrayInit(LIST_LENGTH(pScanCols), POINTER_BYTES); + if (NULL == pArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNode* pCol = NULL; + FOREACH(pCol, pScanCols) { + taosArrayPush(pArray, &pCol); + } + taosArraySort(pArray, colIdCompare); + + int32_t index = 0; + FOREACH(pCol, pScanCols) { + REPLACE_NODE(taosArrayGetP(pArray, index++)); + } + taosArrayDestroy(pArray); + + return TSDB_CODE_SUCCESS; +} + static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) { if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanPhysiNode) || QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN == nodeType(pScanPhysiNode)) { pScanPhysiNode->pScanCols = nodesMakeList(); - CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))); + if (NULL == pScanPhysiNode->pScanCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))) { + return TSDB_CODE_OUT_OF_MEMORY; + } SNode* pNode; FOREACH(pNode, pScanCols) { @@ -228,28 +374,29 @@ static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhys strcpy(pCol->colName, ((SColumnNode*)pNode)->colName); continue; } - CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))); + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))) { + return TSDB_CODE_OUT_OF_MEMORY; + } } } else { pScanPhysiNode->pScanCols = nodesCloneList(pScanCols); - CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pScanPhysiNode->pScanCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - return TSDB_CODE_SUCCESS; + return sortScanCols(pScanPhysiNode->pScanCols); } static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode, SPhysiNode** pPhyNode) { int32_t code = createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols); if (TSDB_CODE_SUCCESS == code) { // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t - code = addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode); } - if (TSDB_CODE_SUCCESS == code) { - code = setSlotOutput(pCxt, pScanLogicNode->node.pTargets, pScanPhysiNode->node.pOutputDataBlockDesc); - } if (TSDB_CODE_SUCCESS == code) { pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; @@ -274,7 +421,7 @@ static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAdd } static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); + STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); if (NULL == pTagScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -282,7 +429,7 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* p } static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); if (NULL == pTableScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -298,20 +445,19 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp } static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - SSystemTableScanPhysiNode* pScan = (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); + SSystemTableScanPhysiNode* pScan = (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } + pScan->showRewrite = pScanLogicNode->showRewrite; + pScan->accountId = pCxt->pPlanCxt->acctId; if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLES)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { - for (int32_t i = 0; i < pScanLogicNode->pVgroupList->numOfVgroups; ++i) { - SQueryNodeAddr addr; - vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups + i, &addr); - taosArrayPush(pCxt->pExecNodeList, &addr); - } + SQueryNodeAddr addr = { .nodeId = MND_VGID, .epSet = pCxt->pPlanCxt->mgmtEpSet }; + taosArrayPush(pCxt->pExecNodeList, &addr); } pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); @@ -320,7 +466,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* } static int32_t createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -384,7 +530,7 @@ static int32_t createJoinOutputCols(SPhysiPlanContext* pCxt, SDataBlockDescNode* } static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode, SPhysiNode** pPhyNode) { - SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN); + SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_JOIN); if (NULL == pJoin) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -397,14 +543,11 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren code = createJoinOutputCols(pCxt, pLeftDesc, pRightDesc, &pJoin->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); } - if (TSDB_CODE_SUCCESS == code) { - code = setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, pJoin->node.pOutputDataBlockDesc); - } if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pJoin; @@ -424,7 +567,9 @@ typedef struct SRewritePrecalcExprsCxt { static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { SNode* pExpr = nodesCloneNode(*pNode); - CHECK_ALLOC(pExpr, DEAL_RES_ERROR); + if (NULL == pExpr) { + return DEAL_RES_ERROR; + } if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) { nodesDestroyNode(pExpr); return DEAL_RES_ERROR; @@ -472,11 +617,15 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN if (NULL == *pPrecalcExprs) { *pPrecalcExprs = nodesMakeList(); - CHECK_ALLOC(*pPrecalcExprs, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == *pPrecalcExprs) { + return TSDB_CODE_OUT_OF_MEMORY; + } } if (NULL == *pRewrittenList) { *pRewrittenList = nodesMakeList(); - CHECK_ALLOC(*pRewrittenList, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == *pRewrittenList) { + return TSDB_CODE_OUT_OF_MEMORY; + } } SNode* pNode = NULL; FOREACH(pNode, pList) { @@ -486,8 +635,12 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } else { pNew = nodesCloneNode(pNode); } - CHECK_ALLOC(pNew, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE(nodesListAppend(*pRewrittenList, pNew), TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pNew) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS != nodesListAppend(*pRewrittenList, pNew)) { + return TSDB_CODE_OUT_OF_MEMORY; + } } SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); @@ -499,7 +652,7 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode, SPhysiNode** pPhyNode) { - SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG); + SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pAggLogicNode, QUERY_NODE_PHYSICAL_PLAN_AGG); if (NULL == pAgg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -517,30 +670,27 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pAgg->pExprs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pAgg->pExprs, pChildTupe); + code = pushdownDataBlockSlots(pCxt, pAgg->pExprs, pChildTupe); } } if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pGroupKeys, &pAgg->pGroupKeys); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pAgg->pGroupKeys, pAgg->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pAgg->pGroupKeys, pAgg->node.pOutputDataBlockDesc); } } if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pAggFuncs, &pAgg->pAggFuncs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pAgg->pAggFuncs, pAgg->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pAgg->pAggFuncs, pAgg->node.pOutputDataBlockDesc); } } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg); } - if (TSDB_CODE_SUCCESS == code) { - code = setSlotOutput(pCxt, pAggLogicNode->node.pTargets, pAgg->node.pOutputDataBlockDesc); - } if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pAgg; @@ -548,18 +698,22 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, nodesDestroyNode(pAgg); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pGroupKeys); + nodesDestroyList(pAggFuncs); + return code; } static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) { - SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT); + SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pProjectLogicNode, QUERY_NODE_PHYSICAL_PLAN_PROJECT); if (NULL == pProject) { return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections, &pProject->pProjections); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pProject->pProjections, pProject->node.pOutputDataBlockDesc); + code = addDataBlockSlotsForProject(pCxt, pProjectLogicNode->stmtName, pProject->pProjections, pProject->node.pOutputDataBlockDesc); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject); @@ -575,34 +729,30 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild } static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { - SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; } pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; - int32_t code = addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc); + *pPhyNode = (SPhysiNode*)pExchange; - if (TSDB_CODE_SUCCESS == code) { - *pPhyNode = (SPhysiNode*)pExchange; - } else { - nodesDestroyNode(pExchange); - } - - return code; + return TSDB_CODE_SUCCESS; } static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { - SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } - int32_t code = addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pScan->node.pOutputDataBlockDesc); + int32_t code = TSDB_CODE_SUCCESS; + + pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets); + if (NULL == pScan->pScanCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } if (TSDB_CODE_SUCCESS == code) { - pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets); - if (NULL == pScan->pScanCols) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = addDataBlockSlots(pCxt, pScan->pScanCols, pScan->node.pOutputDataBlockDesc); } if (TSDB_CODE_SUCCESS == code) { @@ -632,21 +782,17 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pWindow->pExprs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pWindow->pExprs, pChildTupe); + code = addDataBlockSlots(pCxt, pWindow->pExprs, pChildTupe); } } if (TSDB_CODE_SUCCESS == code && NULL != pFuncs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncs, &pWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pWindow->pFuncs, pWindow->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pWindow->pFuncs, pWindow->node.pOutputDataBlockDesc); } } - if (TSDB_CODE_SUCCESS == code) { - code = setSlotOutput(pCxt, pWindowLogicNode->node.pTargets, pWindow->node.pOutputDataBlockDesc); - } - if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pWindow; } else { @@ -657,7 +803,7 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* } static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { - SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_INTERVAL); + SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pWindowLogicNode, QUERY_NODE_PHYSICAL_PLAN_INTERVAL); if (NULL == pInterval) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -674,11 +820,18 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil return TSDB_CODE_OUT_OF_MEMORY; } + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + int32_t code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pTspk, &pInterval->pTspk); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pInterval); + return code; + } + return createWindowPhysiNodeFinalize(pCxt, pChildren, &pInterval->window, pWindowLogicNode, pPhyNode); } static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { - SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW); + SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pWindowLogicNode, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW); if (NULL == pSession) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -702,6 +855,41 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr return TSDB_CODE_FAILED; } +static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode, SPhysiNode** pPhyNode) { + SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pPrecalcExprs = NULL; + SNodeList* pSortKeys = NULL; + int32_t code = rewritePrecalcExprs(pCxt, pSortLogicNode->pSortKeys, &pPrecalcExprs, &pSortKeys); + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + // push down expression to pOutputDataBlockDesc of child node + if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pSort->pExprs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pSort->pExprs, pChildTupe); + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pSortKeys, &pSort->pSortKeys); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pSort->pSortKeys, pSort->node.pOutputDataBlockDesc); + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pSort; + } else { + nodesDestroyNode(pSort); + } + + return code; +} + static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { switch (nodeType(pLogicNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: @@ -716,6 +904,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return createExchangePhysiNode(pCxt, (SExchangeLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: return createWindowPhysiNode(pCxt, pChildren, (SWindowLogicNode*)pLogicNode, pPhyNode); + case QUERY_NODE_LOGIC_PLAN_SORT: + return createSortPhysiNode(pCxt, pChildren, (SSortLogicNode*)pLogicNode, pPhyNode); default: break; } @@ -847,17 +1037,22 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t l SNodeListNode* pGroup; if (level >= LIST_LENGTH(pSubplans)) { pGroup = nodesMakeNode(QUERY_NODE_NODE_LIST); - CHECK_ALLOC(pGroup, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE(nodesListStrictAppend(pSubplans, pGroup), TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pGroup) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pSubplans, pGroup)) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { pGroup = nodesListGetNode(pSubplans, level); } if (NULL == pGroup->pNodeList) { pGroup->pNodeList = nodesMakeList(); - CHECK_ALLOC(pGroup->pNodeList, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pGroup->pNodeList) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - CHECK_CODE(nodesListStrictAppend(pGroup->pNodeList, pSubplan), TSDB_CODE_OUT_OF_MEMORY); - return TSDB_CODE_SUCCESS; + return nodesListStrictAppend(pGroup->pNodeList, pSubplan); } static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent, SQueryPlan* pQueryPlan) { diff --git a/source/libs/planner/src/planScaleOut.c b/source/libs/planner/src/planScaleOut.c index 6928b1728f..ca6c7a2577 100644 --- a/source/libs/planner/src/planScaleOut.c +++ b/source/libs/planner/src/planScaleOut.c @@ -61,7 +61,7 @@ static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, static int32_t doSetScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { SScanLogicNode* pScan = (SScanLogicNode*)pNode; - pScan->pVgroupList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + pScan->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); if (NULL == pScan->pVgroupList) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index d203d41cb0..df546c32be 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -64,8 +64,10 @@ static int32_t stsMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan) { } SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { - SStsInfo* pInfo = calloc(1, sizeof(SStsInfo)); - CHECK_ALLOC(pInfo, TSDB_CODE_OUT_OF_MEMORY); + SStsInfo* pInfo = taosMemoryCalloc(1, sizeof(SStsInfo)); + if (NULL == pInfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } pInfo->pScan = (SScanLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; pCxt->pInfo = pInfo; diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index 69203384f8..3d17cc260b 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -112,6 +112,11 @@ private: if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; pCxt->topicQuery = true; + } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { + SMCreateSmaReq req = {0}; + tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); + nodesStringToNode(req.ast, &pCxt->pAstRoot); + pCxt->streamQuery = true; } else { pCxt->pAstRoot = pQuery->pRoot; } @@ -133,7 +138,7 @@ private: return string(); } string str(pStr); - tfree(pStr); + taosMemoryFreeClear(pStr); return str; } @@ -165,7 +170,7 @@ TEST_F(PlannerTest, groupBy) { bind("SELECT count(*) FROM t1"); ASSERT_TRUE(run()); - bind("SELECT c1, count(*) FROM t1 GROUP BY c1"); + bind("SELECT c1, max(c3), min(c2), count(*) FROM t1 GROUP BY c1"); ASSERT_TRUE(run()); bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3"); @@ -196,10 +201,31 @@ TEST_F(PlannerTest, sessionWindow) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, orderBy) { + setDatabase("root", "test"); + + bind("SELECT * FROM t1 order by c1"); + ASSERT_TRUE(run()); + + bind("SELECT c1 FROM t1 order by c2"); + ASSERT_TRUE(run()); + + bind("SELECT * FROM t1 order by c1 + 10, c2"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, showTables) { setDatabase("root", "test"); bind("show tables"); + ASSERT_TRUE(run()); +} + +TEST_F(PlannerTest, showStables) { + setDatabase("root", "test"); + + bind("show stables"); + ASSERT_TRUE(run()); } TEST_F(PlannerTest, createTopic) { @@ -215,3 +241,10 @@ TEST_F(PlannerTest, stream) { bind("SELECT sum(c1) FROM st1"); ASSERT_TRUE(run(true)); } + +TEST_F(PlannerTest, createSmaIndex) { + setDatabase("root", "test"); + + bind("create sma index index1 on t1 function(max(c1), min(c3 + 10), sum(c4)) INTERVAL(10s)"); + ASSERT_TRUE(run()); +} diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 79a59a5ecb..4ac19294aa 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -140,7 +140,7 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) return 0; } -int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { +int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *rpcCtx) { char* pMsg = rpcMallocCont(pInfo->msgInfo.len); if (NULL == pMsg) { qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); @@ -154,6 +154,7 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp .contLen = pInfo->msgInfo.len, .ahandle = (void*)pInfo, .handle = pInfo->msgInfo.handle, + .persistHandle = persistHandle, .code = 0}; if (pInfo->msgType == TDMT_VND_QUERY || pInfo->msgType == TDMT_VND_FETCH || pInfo->msgType == TDMT_VND_QUERY_CONTINUE) { @@ -162,10 +163,14 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp assert(pInfo->fp != NULL); - rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId); + rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx); return TSDB_CODE_SUCCESS; } +int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { + return asyncSendMsgToServerExt(pTransporter, epSet, pTransporterId, pInfo, false, NULL); +} + char *jobTaskStatusStr(int32_t status) { switch (status) { case JOB_TASK_STATUS_NULL: @@ -193,7 +198,7 @@ char *jobTaskStatusStr(int32_t status) { return "UNKNOWN"; } -SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name) { +SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name) { SSchema s = {0}; s.type = type; s.bytes = bytes; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index b4763024dc..1e91c55dc0 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -28,7 +28,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { memcpy(pOut->db, usedbRsp->db, TSDB_DB_FNAME_LEN); pOut->dbId = usedbRsp->uid; - pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); + pOut->dbVgroup = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -151,7 +151,7 @@ PROCESS_USEDB_OVER: if (code != 0) { if (pOut) { if (pOut->dbVgroup) taosHashCleanup(pOut->dbVgroup->vgHash); - tfree(pOut->dbVgroup); + taosMemoryFreeClear(pOut->dbVgroup); } qError("failed to process usedb rsp since %s", terrstr()); } @@ -188,7 +188,7 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp *pMetaMsg) { } if (pMetaMsg->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchemas[0].colId); + qError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", pMetaMsg->pSchemas[0].colId); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -199,7 +199,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STabl int32_t total = msg->numOfColumns + msg->numOfTags; int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; - STableMeta *pTableMeta = calloc(1, metaSize); + STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { qError("calloc size[%d] failed", metaSize); return TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp index 72ce0f7c37..9615557c88 100644 --- a/source/libs/qcom/test/queryTest.cpp +++ b/source/libs/qcom/test/queryTest.cpp @@ -32,13 +32,13 @@ typedef struct SParam { int32_t testPrint(void* p) { SParam* param = (SParam*)p; printf("hello world, %d\n", param->v); - tfree(p); + taosMemoryFreeClear(p); return 0; } int32_t testPrintError(void* p) { SParam* param = (SParam*) p; - tfree(p); + taosMemoryFreeClear(p); return -1; } @@ -61,14 +61,14 @@ int main(int argc, char** argv) { } TEST(testCase, async_task_test) { - SParam* p = (SParam*)calloc(1, sizeof(SParam)); + SParam* p = (SParam*)taosMemoryCalloc(1, sizeof(SParam)); taosAsyncExec(testPrint, p, NULL); taosMsleep(5); } TEST(testCase, many_async_task_test) { for(int32_t i = 0; i < 50; ++i) { - SParam* p = (SParam*) calloc(1, sizeof(SParam)); + SParam* p = (SParam*) taosMemoryCalloc(1, sizeof(SParam)); p->v = i; taosAsyncExec(testPrint, p, NULL); } @@ -78,7 +78,7 @@ TEST(testCase, many_async_task_test) { TEST(testCase, error_in_async_test) { int32_t code = 0; - SParam* p = (SParam*) calloc(1, sizeof(SParam)); + SParam* p = (SParam*) taosMemoryCalloc(1, sizeof(SParam)); taosAsyncExec(testPrintError, p, &code); taosMsleep(1); printf("Error code:%d after asynchronously exec function\n", code); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index e768aa2de0..573eaed2e6 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -69,18 +69,24 @@ enum { typedef struct SQWDebug { bool lockEnable; bool statusEnable; + bool dumpEnable; } SQWDebug; +typedef struct SQWConnInfo { + void *handle; + void *ahandle; +} SQWConnInfo; + typedef struct SQWMsg { - void *node; - char *msg; - int32_t msgLen; - void *connection; + void *node; + char *msg; + int32_t msgLen; + SQWConnInfo connInfo; } SQWMsg; typedef struct SQWHbInfo { SSchedulerHbRsp rsp; - void *connection; + SQWConnInfo connInfo; } SQWHbInfo; typedef struct SQWPhaseInput { @@ -101,10 +107,6 @@ typedef struct SQWTaskCtx { SRWLatch lock; int8_t phase; int8_t taskType; - - void *readyConnection; - void *dropConnection; - void *cancelConnection; bool emptyRes; bool queryFetched; @@ -113,6 +115,7 @@ typedef struct SQWTaskCtx { bool queryInQueue; int32_t rspCode; + SQWConnInfo connInfo; int8_t events[QW_EVENT_MAX]; qTaskInfo_t taskHandle; @@ -120,11 +123,12 @@ typedef struct SQWTaskCtx { } SQWTaskCtx; typedef struct SQWSchStatus { - int32_t lastAccessTs; // timestamp in second - uint64_t hbSeqId; - void *hbConnection; - SRWLatch tasksLock; - SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus + int32_t lastAccessTs; // timestamp in second + SRWLatch hbConnLock; + SQWConnInfo hbConnInfo; + SQueryNodeEpId hbEpId; + SRWLatch tasksLock; + SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus } SQWSchStatus; // Qnode/Vnode level task management @@ -172,12 +176,16 @@ typedef struct SQWorkerMgmt { #define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__) #define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__) +#define QW_DUMP(param, ...) do { if (gQWDebug.dumpEnable) { qDebug("QW:%p " param, mgmt, __VA_ARGS__); } } while (0) + + #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) #define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOGL(param, ...) qDebugL("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) #define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) @@ -223,8 +231,6 @@ typedef struct SQWorkerMgmt { } \ } while (0) -int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code); - #ifdef __cplusplus } #endif diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index ecb5dbd654..be1d47a189 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -30,17 +30,18 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req); -int32_t qwBuildAndSendDropRsp(void *connection, int32_t code); -int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code); -int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); +int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code); +int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code); +int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete); -int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection); -int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code); -int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code); +int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn); +int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code); +int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code); void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp); -int32_t qwBuildAndSendHbRsp(SRpcMsg *pMsg, SSchedulerHbRsp *rsp, int32_t code); +int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); +int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ae2dccfe39..71e7415ea5 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -9,7 +9,7 @@ #include "tname.h" #include "dataSinkMgt.h" -SQWDebug gQWDebug = {.statusEnable = true}; +SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { if (!gQWDebug.statusEnable) { @@ -103,6 +103,36 @@ _return: QW_RET(code); } +void qwDbgDumpSchInfo(SQWSchStatus *sch, int32_t i) { + +} + +void qwDbgDumpMgmtInfo(SQWorkerMgmt *mgmt) { + if (!gQWDebug.dumpEnable) { + return; + } + + QW_LOCK(QW_READ, &mgmt->schLock); + + QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash)); + + void *key = NULL; + size_t keyLen = 0; + int32_t i = 0; + SQWSchStatus *sch = NULL; + + void *pIter = taosHashIterate(mgmt->schHash, NULL); + while (pIter) { + sch = (SQWSchStatus *)pIter; + qwDbgDumpSchInfo(sch, i); + ++i; + pIter = taosHashIterate(mgmt->schHash, pIter); + } + + QW_UNLOCK(QW_READ, &mgmt->schLock); + + QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash)); +} char *qwPhaseStr(int32_t phase) { switch (phase) { @@ -166,7 +196,7 @@ int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status) { } -int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) { +int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { @@ -200,7 +230,7 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, QW_UNLOCK(rwType, &mgmt->schLock); if (QW_NOT_EXIST_ADD == nOpt) { - QW_ERR_RET(qwAddSchedulerImpl(mgmt, sId, rwType, sch)); + QW_ERR_RET(qwAddSchedulerImpl(mgmt, sId, rwType)); nOpt = QW_NOT_EXIST_RET_ERR; @@ -402,6 +432,9 @@ int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { + rpcReleaseHandle(ctx->connInfo.handle, TAOS_CONN_SERVER); + ctx->connInfo.handle = NULL; + qwFreeTaskHandle(QW_FPARAMS(), &ctx->taskHandle); if (ctx->sinkHandle) { @@ -516,7 +549,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { while (true) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); - + code = qExecTask(*taskHandle, &pRes, &useconds); if (code) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -577,6 +610,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; + hbInfo->connInfo = sch->hbConnInfo; + hbInfo->rsp.epId = sch->hbEpId; + QW_LOCK(QW_READ, &sch->tasksLock); taskNum = taosHashGetSize(sch->tasksHash); @@ -588,9 +624,6 @@ int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbI return TSDB_CODE_QRY_OUT_OF_MEMORY; } - hbInfo->connection = sch->hbConnection; - hbInfo->rsp.seqId = -1; - void *key = NULL; size_t keyLen = 0; int32_t i = 0; @@ -694,8 +727,8 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; SQWTaskCtx *ctx = NULL; - void *dropConnection = NULL; - void *cancelConnection = NULL; + SQWConnInfo *dropConnection = NULL; + SQWConnInfo *cancelConnection = NULL; QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase)); @@ -727,9 +760,13 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + dropConnection = &ctx->connInfo; QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + dropConnection = NULL; + + qwBuildAndSendDropRsp(&ctx->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code)); - dropConnection = ctx->dropConnection; QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); break; } @@ -761,9 +798,13 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + dropConnection = &ctx->connInfo; QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + dropConnection = NULL; - dropConnection = ctx->dropConnection; + qwBuildAndSendDropRsp(&ctx->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code)); + QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } @@ -789,13 +830,13 @@ _return: } if (dropConnection) { - qwBuildAndSendDropRsp(dropConnection, code); - QW_TASK_DLOG("drop msg rsped, code:%x - %s", code, tstrerror(code)); + qwBuildAndSendDropRsp(dropConnection, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", dropConnection->handle, code, tstrerror(code)); } if (cancelConnection) { - qwBuildAndSendCancelRsp(cancelConnection, code); - QW_TASK_DLOG("cancel msg rsped, code:%x - %s", code, tstrerror(code)); + qwBuildAndSendCancelRsp(cancelConnection, code); + QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code)); } QW_TASK_DLOG("end to handle event at phase %s, code:%x - %s", qwPhaseStr(phase), code, tstrerror(code)); @@ -807,9 +848,8 @@ _return: int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; SQWTaskCtx *ctx = NULL; - void *readyConnection = NULL; - void *dropConnection = NULL; - void *cancelConnection = NULL; + SQWConnInfo connInfo = {0}; + SQWConnInfo *readyConnection = NULL; QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase)); @@ -826,11 +866,18 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) { ctx->emptyRes = true; } - + +#if 0 if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) { - readyConnection = ctx->readyConnection; + readyConnection = &ctx->connInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); } +#else + connInfo.handle = ctx->connInfo.handle; + readyConnection = &connInfo; + + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); +#endif } if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { @@ -838,10 +885,11 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } + + qwBuildAndSendDropRsp(&ctx->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code)); QW_ERR_JRET(qwDropTask(QW_FPARAMS())); - - dropConnection = ctx->dropConnection; QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } @@ -870,18 +918,8 @@ _return: } if (readyConnection) { - qwBuildAndSendReadyRsp(readyConnection, code); - QW_TASK_DLOG("ready msg rsped, code:%x - %s", code, tstrerror(code)); - } - - if (dropConnection) { - qwBuildAndSendDropRsp(dropConnection, code); - QW_TASK_DLOG("drop msg rsped, code:%x - %s", code, tstrerror(code)); - } - - if (cancelConnection) { - qwBuildAndSendCancelRsp(cancelConnection, code); - QW_TASK_DLOG("cancel msg rsped, code:%x - %s", code, tstrerror(code)); + qwBuildAndSendReadyRsp(readyConnection, code); + QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", readyConnection->handle, code, tstrerror(code)); } if (code) { @@ -893,23 +931,28 @@ _return: QW_RET(code); } - int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { int32_t code = 0; bool queryRsped = false; - bool needStop = false; struct SSubplan *plan = NULL; SQWPhaseInput input = {0}; qTaskInfo_t pTaskInfo = NULL; DataSinkHandle sinkHandle = NULL; SQWTaskCtx *ctx = NULL; + QW_ERR_JRET(qwRegisterBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); atomic_store_8(&ctx->taskType, taskType); - + + atomic_store_ptr(&ctx->connInfo.handle, qwMsg->connInfo.handle); + atomic_store_ptr(&ctx->connInfo.ahandle, qwMsg->connInfo.ahandle); + + QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); + code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { QW_TASK_ELOG("task string to subplan failed, code:%x - %s", code, tstrerror(code)); @@ -927,8 +970,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - QW_ERR_JRET(qwBuildAndSendQueryRsp(qwMsg->connection, code)); - QW_TASK_DLOG("query msg rsped, code:%x - %s", code, tstrerror(code)); + QW_ERR_JRET(qwBuildAndSendQueryRsp(&qwMsg->connInfo, code)); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); queryRsped = true; @@ -945,11 +988,11 @@ _return: code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); if (!queryRsped) { - qwBuildAndSendQueryRsp(qwMsg->connection, code); - QW_TASK_DLOG("query msg rsped, code:%x - %s", code, tstrerror(code)); + qwBuildAndSendQueryRsp(&qwMsg->connInfo, code); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); } - QW_RET(code); + QW_RET(TSDB_CODE_SUCCESS); } int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) { @@ -968,8 +1011,9 @@ int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } if (ctx->phase == QW_PHASE_PRE_QUERY) { + ctx->connInfo.handle == qwMsg->connInfo.handle; + ctx->connInfo.ahandle = qwMsg->connInfo.ahandle; QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_READY); - ctx->readyConnection = qwMsg->connection; needRsp = false; QW_TASK_DLOG_E("ready msg will not rsp now"); goto _return; @@ -1007,11 +1051,11 @@ _return: } if (needRsp) { - qwBuildAndSendReadyRsp(qwMsg->connection, code); - QW_TASK_DLOG("ready msg rsped, code:%x - %s", code, tstrerror(code)); + qwBuildAndSendReadyRsp(&qwMsg->connInfo, code); + QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); } - QW_RET(code); + QW_RET(TSDB_CODE_SUCCESS); } @@ -1048,10 +1092,11 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); atomic_store_8((int8_t*)&ctx->queryEnd, qComplete); + qwMsg->connInfo = ctx->connInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); - QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); + qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen); } else { atomic_store_8((int8_t*)&ctx->queryContinue, 1); } @@ -1067,8 +1112,10 @@ _return: QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwFreeFetchRsp(rsp); rsp = NULL; - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, 0, code); - QW_TASK_DLOG("fetch msg rsped, code:%x - %s", code, tstrerror(code)); + + qwMsg->connInfo = ctx->connInfo; + qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, 0, code); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0); } QW_LOCK(QW_WRITE, &ctx->lock); @@ -1082,7 +1129,9 @@ _return: } while (true); input.code = code; - QW_RET(qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL)); + qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL); + + QW_RET(TSDB_CODE_SUCCESS); } @@ -1097,11 +1146,14 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - + SOutputData sOutput = {0}; QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); if (NULL == rsp) { + atomic_store_ptr(&ctx->connInfo.handle, qwMsg->connInfo.handle); + atomic_store_ptr(&ctx->connInfo.ahandle, qwMsg->connInfo.ahandle); + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); } else { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); @@ -1123,7 +1175,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { atomic_store_8((int8_t*)&ctx->queryInQueue, 1); - QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection)); + QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), &qwMsg->connInfo)); } } @@ -1143,17 +1195,17 @@ _return: } if (code || rsp) { - qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code); - QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen); + qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen); } - QW_RET(code); + QW_RET(TSDB_CODE_SUCCESS); } int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; - bool needRsp = false; + bool rsped = false; SQWTaskCtx *ctx = NULL; bool locked = false; @@ -1174,14 +1226,18 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING); } else if (ctx->phase > 0) { + qwBuildAndSendDropRsp(&qwMsg->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); + QW_ERR_JRET(qwDropTask(QW_FPARAMS())); - needRsp = true; + rsped = true; } else { // task not started } - if (!needRsp) { - ctx->dropConnection = qwMsg->connection; + if (!rsped) { + ctx->connInfo.handle = qwMsg->connInfo.handle; + ctx->connInfo.ahandle = qwMsg->connInfo.ahandle; QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } @@ -1204,13 +1260,12 @@ _return: qwReleaseTaskCtx(mgmt, ctx); } - if (TSDB_CODE_SUCCESS != code || needRsp) { - QW_ERR_RET(qwBuildAndSendDropRsp(qwMsg->connection, code)); - - QW_TASK_DLOG("drop msg rsped, code:%x", code); + if (TSDB_CODE_SUCCESS != code) { + qwBuildAndSendDropRsp(&qwMsg->connInfo, code); + QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); } - QW_RET(code); + QW_RET(TSDB_CODE_SUCCESS); } int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { @@ -1218,38 +1273,46 @@ int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { SSchedulerHbRsp rsp = {0}; SQWSchStatus *sch = NULL; uint64_t seqId = 0; + void *origHandle = NULL; memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); QW_ERR_JRET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); - atomic_store_ptr(&sch->hbConnection, qwMsg->connection); - ++sch->hbSeqId; + QW_LOCK(QW_WRITE, &sch->hbConnLock); - rsp.seqId = sch->hbSeqId; - - QW_DLOG("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, connection:%p", - sch->hbSeqId, req->sId, req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connection); + if (sch->hbConnInfo.handle) { + rpcReleaseHandle(sch->hbConnInfo.handle, TAOS_CONN_SERVER); + } + + memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); + memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); + + QW_UNLOCK(QW_WRITE, &sch->hbConnLock); + + QW_DLOG("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, handle:%p, ahandle:%p", + req->sId, req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connInfo.handle, qwMsg->connInfo.ahandle); qwReleaseScheduler(QW_READ, mgmt); _return: - qwBuildAndSendHbRsp(qwMsg->connection, &rsp, code); + qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code); + QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); - QW_RET(code); + QW_RET(TSDB_CODE_SUCCESS); } void qwProcessHbTimerEvent(void *param, void *tmrId) { - return; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)param; SQWSchStatus *sch = NULL; int32_t taskNum = 0; SQWHbInfo *rspList = NULL; int32_t code = 0; + qwDbgDumpMgmtInfo(mgmt); + QW_LOCK(QW_READ, &mgmt->schLock); int32_t schNum = taosHashGetSize(mgmt->schHash); @@ -1259,7 +1322,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { return; } - rspList = calloc(schNum, sizeof(SQWHbInfo)); + rspList = taosMemoryCalloc(schNum, sizeof(SQWHbInfo)); if (NULL == rspList) { QW_UNLOCK(QW_READ, &mgmt->schLock); QW_ELOG("calloc %d SQWHbInfo failed", schNum); @@ -1288,12 +1351,12 @@ _return: QW_UNLOCK(QW_READ, &mgmt->schLock); for (int32_t j = 0; j < i; ++j) { - QW_DLOG("hb on connection %p, taskNum:%d", rspList[j].connection, (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0)); - qwBuildAndSendHbRsp(rspList[j].connection, &rspList[j].rsp, code); + qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); + QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code), (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0)); tFreeSSchedulerHbRsp(&rspList[j].rsp); } - tfree(rspList); + taosMemoryFreeClear(rspList); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); } @@ -1305,7 +1368,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW } int32_t code = 0; - SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); + SQWorkerMgmt *mgmt = taosMemoryCalloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1330,7 +1393,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == mgmt->schHash) { - tfree(mgmt); + taosMemoryFreeClear(mgmt); qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum); QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1370,7 +1433,7 @@ _return: taosTmrCleanUp(mgmt->timer); - tfree(mgmt); + taosMemoryFreeClear(mgmt); QW_RET(code); } @@ -1389,7 +1452,7 @@ void qWorkerDestroy(void **qWorkerMgmt) { //TODO FREE ALL - tfree(*qWorkerMgmt); + taosMemoryFreeClear(*qWorkerMgmt); } int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp) { @@ -1406,7 +1469,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs taskNum = taosHashGetSize(sch->tasksHash); int32_t size = sizeof(SSchedulerStatusRsp) + sizeof((*rsp)->status[0]) * taskNum; - *rsp = calloc(1, size); + *rsp = taosMemoryCalloc(1, size); if (NULL == *rsp) { QW_SCH_ELOG("calloc %d failed", size); QW_UNLOCK(QW_READ, &sch->tasksLock); diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 79832b4db4..ff9527f7b9 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -26,6 +26,8 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { return TSDB_CODE_SUCCESS; } + + void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; @@ -44,8 +46,7 @@ void qwFreeFetchRsp(void *msg) { } } -int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) { - SRpcMsg *pMsg = (SRpcMsg *)connection; +int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) { SQueryTableRsp rsp = {.code = code}; int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp); @@ -54,8 +55,8 @@ int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) { SRpcMsg rpcRsp = { .msgType = TDMT_VND_QUERY_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .pCont = msg, .contLen = contLen, .code = code, @@ -66,15 +67,14 @@ int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) { return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code) { - SRpcMsg *pMsg = (SRpcMsg *)connection; +int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) { SResReadyRsp *pRsp = (SResReadyRsp *)rpcMallocCont(sizeof(SResReadyRsp)); pRsp->code = code; SRpcMsg rpcRsp = { .msgType = TDMT_VND_RES_READY_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = NULL, .pCont = pRsp, .contLen = sizeof(*pRsp), .code = code, @@ -85,15 +85,15 @@ int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code) { return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendHbRsp(SRpcMsg *pMsg, SSchedulerHbRsp *pStatus, int32_t code) { +int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) { int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus); void *pRsp = rpcMallocCont(contLen); tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus); SRpcMsg rpcRsp = { .msgType = TDMT_VND_QUERY_HEARTBEAT_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .pCont = pRsp, .contLen = contLen, .code = code, @@ -104,9 +104,7 @@ int32_t qwBuildAndSendHbRsp(SRpcMsg *pMsg, SSchedulerHbRsp *pStatus, int32_t cod return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) { - SRpcMsg *pMsg = (SRpcMsg *)connection; - +int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) { if (NULL == pRsp) { pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRsp, 0, sizeof(SRetrieveTableRsp)); @@ -115,8 +113,8 @@ int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_ SRpcMsg rpcRsp = { .msgType = TDMT_VND_FETCH_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .pCont = pRsp, .contLen = sizeof(*pRsp) + dataLength, .code = code, @@ -127,14 +125,14 @@ int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_ return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { +int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code) { STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp)); pRsp->code = code; SRpcMsg rpcRsp = { .msgType = TDMT_VND_CANCEL_TASK_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .pCont = pRsp, .contLen = sizeof(*pRsp), .code = code, @@ -144,15 +142,14 @@ int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendDropRsp(void *connection, int32_t code) { - SRpcMsg *pMsg = (SRpcMsg *)connection; +int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code) { STaskDropRsp *pRsp = (STaskDropRsp *)rpcMallocCont(sizeof(STaskDropRsp)); pRsp->code = code; SRpcMsg rpcRsp = { .msgType = TDMT_VND_DROP_TASK_RSP, - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, + .handle = pConn->handle, + .ahandle = pConn->ahandle, .pCont = pRsp, .contLen = sizeof(*pRsp), .code = code, @@ -167,13 +164,13 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { SVShowTablesRsp showRsp = {0}; // showRsp.showId = 1; - showRsp.tableMeta.pSchemas = calloc(numOfCols, sizeof(SSchema)); + showRsp.tableMeta.pSchemas = taosMemoryCalloc(numOfCols, sizeof(SSchema)); if (showRsp.tableMeta.pSchemas == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - int32_t cols = 0; + col_id_t cols = 0; SSchema *pSchema = showRsp.tableMeta.pSchemas; const SSchema *s = tGetTbnameColumnSchema(); @@ -234,8 +231,7 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection) { - SRpcMsg *pMsg = (SRpcMsg *)connection; +int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); @@ -248,12 +244,12 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection) { req->taskId = tId; SRpcMsg pNewMsg = { - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, - .msgType = TDMT_VND_QUERY_CONTINUE, - .pCont = req, - .contLen = sizeof(SQueryContinueReq), - .code = 0, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .msgType = TDMT_VND_QUERY_CONTINUE, + .pCont = req, + .contLen = sizeof(SQueryContinueReq), + .code = 0, }; int32_t code = tmsgPutToQueue(&mgmt->msgCb, QUERY_QUEUE, &pNewMsg); @@ -268,6 +264,35 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection) { return TSDB_CODE_SUCCESS; } + +int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { + STaskDropReq * req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq)); + if (NULL == req) { + QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(STaskDropReq)); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + req->header.vgId = htonl(mgmt->nodeId); + req->sId = htobe64(sId); + req->queryId = htobe64(qId); + req->taskId = htobe64(tId); + req->refId = htobe64(rId); + + SRpcMsg pMsg = { + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .msgType = TDMT_VND_DROP_TASK, + .pCont = req, + .contLen = sizeof(STaskDropReq), + .code = TSDB_CODE_RPC_NETWORK_UNAVAIL, + }; + + rpcRegisterBrokenLinkArg(&pMsg); + + return TSDB_CODE_SUCCESS; +} + + int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -294,11 +319,13 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t tId = msg->taskId; int64_t rId = msg->refId; - SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; char* sql = strndup(msg->msg, msg->sqlLen); - QW_SCH_TASK_DLOG("processQuery start, node:%p, sql:%s", node, sql); - tfree(sql); + QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->handle, sql); + taosMemoryFreeClear(sql); QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); @@ -326,9 +353,11 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t tId = msg->taskId; int64_t rId = 0; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; - QW_SCH_TASK_DLOG("processCQuery start, node:%p", node); + QW_SCH_TASK_DLOG("processCQuery start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessCQuery(QW_FPARAMS(), &qwMsg)); @@ -358,9 +387,11 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ uint64_t tId = msg->taskId; int64_t rId = 0; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; - QW_SCH_TASK_DLOG("processReady start, node:%p", node); + QW_SCH_TASK_DLOG("processReady start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessReady(QW_FPARAMS(), &qwMsg)); @@ -418,9 +449,11 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t tId = msg->taskId; int64_t rId = 0; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; - QW_SCH_TASK_DLOG("processFetch start, node:%p", node); + QW_SCH_TASK_DLOG("processFetch start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessFetch(QW_FPARAMS(), &qwMsg)); @@ -439,6 +472,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; int32_t code = 0; STaskCancelReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { @@ -451,11 +485,21 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { msg->taskId = be64toh(msg->taskId); msg->refId = be64toh(msg->refId); + uint64_t sId = msg->sId; + uint64_t qId = msg->queryId; + uint64_t tId = msg->taskId; + int64_t rId = msg->refId; + + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; + //QW_ERR_JRET(qwCancelTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId)); _return: - QW_ERR_RET(qwBuildAndSendCancelRsp(pMsg, code)); + QW_ERR_RET(qwBuildAndSendCancelRsp(&qwMsg.connInfo, code)); + QW_SCH_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", qwMsg.connInfo.handle, code, tstrerror(code)); return TSDB_CODE_SUCCESS; } @@ -484,9 +528,15 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t tId = msg->taskId; int64_t rId = msg->refId; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; - QW_SCH_TASK_DLOG("processDrop start, node:%p", node); + if (TSDB_CODE_RPC_NETWORK_UNAVAIL == pMsg->code) { + QW_SCH_TASK_DLOG("receive drop task due to network broken, error:%s", tstrerror(pMsg->code)); + } + + QW_SCH_TASK_DLOG("processDrop start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessDrop(QW_FPARAMS(), &qwMsg)); @@ -516,9 +566,11 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } uint64_t sId = req.sId; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; + qwMsg.connInfo.handle = pMsg->handle; + qwMsg.connInfo.ahandle = pMsg->ahandle; - QW_SCH_DLOG("processHb start, node:%p", node); + QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessHb(mgmt, &qwMsg, &req)); @@ -535,7 +587,7 @@ int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SVShowTablesReq *pReq = pMsg->pCont; - QW_ERR_RET(qwBuildAndSendShowRsp(pMsg, code)); + QW_RET(qwBuildAndSendShowRsp(pMsg, code)); } int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { @@ -544,7 +596,7 @@ int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) } SVShowTablesFetchReq *pFetchReq = pMsg->pCont; - QW_ERR_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq)); + QW_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq)); } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 36f1e8d900..7179fa6984 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -165,7 +165,7 @@ int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { taosWLockLatch(&qwtTestFetchQueueLock); - struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg)); memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg; if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) { @@ -188,7 +188,7 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { taosWLockLatch(&qwtTestQueryQueueLock); - struct SRpcMsg *newMsg = (struct SRpcMsg *)calloc(1, sizeof(struct SRpcMsg)); + struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg)); memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg; if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { @@ -312,7 +312,7 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { } if (endExec) { - *pRes = (SSDataBlock*)calloc(1, sizeof(SSDataBlock)); + *pRes = (SSDataBlock*)taosMemoryCalloc(1, sizeof(SSDataBlock)); (*pRes)->info.rows = taosRand() % 1000 + 1; } else { *pRes = NULL; @@ -336,7 +336,7 @@ int32_t qwtPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* p assert(0); } - free((void *)pInput->pData); + taosMemoryFree((void *)pInput->pData); taosWLockLatch(&qwtTestSinkLock); @@ -763,7 +763,7 @@ void *queryQueueThread(void *param) { assert(0); } - free(queryRpc); + taosMemoryFree(queryRpc); if (qwtTestStop && qwtTestQueryQueueNum <= 0 && qwtTestCaseFinished) { break; @@ -832,7 +832,7 @@ void *fetchQueueThread(void *param) { break; } - free(fetchRpc); + taosMemoryFree(fetchRpc); if (qwtTestStop && qwtTestFetchQueueNum <= 0 && qwtTestCaseFinished) { break; @@ -887,8 +887,8 @@ TEST(seqTest, normalCase) { code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); ASSERT_EQ(code, 0); - code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); - ASSERT_EQ(code, 0); + //code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); + //ASSERT_EQ(code, 0); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); ASSERT_EQ(code, 0); @@ -985,12 +985,12 @@ TEST(seqTest, randCase) { qwtBuildQueryReqMsg(&queryRpc); code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); } else if (r >= maxr/5 && r < maxr * 2/5) { - printf("Ready,%d\n", t++); - qwtBuildReadyReqMsg(&readyMsg, &readyRpc); - code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); - if (qwtTestEnableSleep) { - taosUsleep(1); - } + //printf("Ready,%d\n", t++); + //qwtBuildReadyReqMsg(&readyMsg, &readyRpc); + //code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); + //if (qwtTestEnableSleep) { + // taosUsleep(1); + //} } else if (r >= maxr * 2/5 && r < maxr* 3/5) { printf("Fetch,%d\n", t++); qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); @@ -1054,7 +1054,7 @@ TEST(seqTest, multithreadRand) { TdThread t1,t2,t3,t4,t5,t6; taosThreadCreate(&(t1), &thattr, queryThread, mgmt); - taosThreadCreate(&(t2), &thattr, readyThread, NULL); + //taosThreadCreate(&(t2), &thattr, readyThread, NULL); taosThreadCreate(&(t3), &thattr, fetchThread, NULL); taosThreadCreate(&(t4), &thattr, dropThread, NULL); taosThreadCreate(&(t5), &thattr, statusThread, NULL); diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index e1ffb2efd1..834a722bd8 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -338,7 +338,7 @@ typedef struct SFilterInfo { #define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) #define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) -#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0) +#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = taosMemoryMalloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0) #define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) diff --git a/source/libs/scalar/inc/sclfunc.h b/source/libs/scalar/inc/sclfunc.h index 8e03470033..8915f37261 100644 --- a/source/libs/scalar/inc/sclfunc.h +++ b/source/libs/scalar/inc/sclfunc.h @@ -37,7 +37,6 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); - #ifdef __cplusplus } #endif diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 58b5c8340a..23e3e352c8 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -297,14 +297,14 @@ static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void int32_t filterInitUnitsFields(SFilterInfo *info) { info->unitSize = FILTER_DEFAULT_UNIT_SIZE; - info->units = calloc(info->unitSize, sizeof(SFilterUnit)); + info->units = taosMemoryCalloc(info->unitSize, sizeof(SFilterUnit)); info->fields[FLD_TYPE_COLUMN].num = 0; info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField)); + info->fields[FLD_TYPE_COLUMN].fields = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField)); info->fields[FLD_TYPE_VALUE].num = 0; info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); + info->fields[FLD_TYPE_VALUE].fields = taosMemoryCalloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); return TSDB_CODE_SUCCESS; } @@ -318,7 +318,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt r->prev = NULL; r->next = NULL; } else { - r = calloc(1, sizeof(SFilterRangeNode)); + r = taosMemoryCalloc(1, sizeof(SFilterRangeNode)); } FILTER_COPY_RA(&r->ra, ra); @@ -332,7 +332,7 @@ void* filterInitRangeCtx(int32_t type, int32_t options) { return NULL; } - SFilterRangeCtx *ctx = calloc(1, sizeof(SFilterRangeCtx)); + SFilterRangeCtx *ctx = taosMemoryCalloc(1, sizeof(SFilterRangeCtx)); ctx->type = type; ctx->options = options; @@ -748,18 +748,18 @@ int32_t filterFreeRangeCtx(void* h) { while (r) { rn = r->next; - free(r); + taosMemoryFree(r); r = rn; } r = ctx->rf; while (r) { rn = r->next; - free(r); + taosMemoryFree(r); r = rn; } - free(ctx); + taosMemoryFree(ctx); return TSDB_CODE_SUCCESS; } @@ -769,7 +769,7 @@ int32_t filterDetachCnfGroup(SFilterGroup *gp1, SFilterGroup *gp2, SArray* group SFilterGroup gp = {0}; gp.unitNum = gp1->unitNum + gp2->unitNum; - gp.unitIdxs = calloc(gp.unitNum, sizeof(*gp.unitIdxs)); + gp.unitIdxs = taosMemoryCalloc(gp.unitNum, sizeof(*gp.unitIdxs)); memcpy(gp.unitIdxs, gp1->unitIdxs, gp1->unitNum * sizeof(*gp.unitIdxs)); memcpy(gp.unitIdxs + gp1->unitNum, gp2->unitIdxs, gp2->unitNum * sizeof(*gp.unitIdxs)); @@ -849,7 +849,7 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d return -1; } -// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0 +// In the params, we should use void *data instead of void **data, there is no need to use taosMemoryFreeClear(*data) to set *data = 0 // Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better. int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { int32_t idx = -1; @@ -869,7 +869,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, idx = *num; if (idx >= info->fields[type].size) { info->fields[type].size += FILTER_DEFAULT_FIELD_SIZE; - info->fields[type].fields = realloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); + info->fields[type].fields = taosMemoryRealloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); } info->fields[type].fields[idx].flag = type; @@ -891,7 +891,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, } } else { if (data && freeIfExists) { - tfree(*data); + taosMemoryFreeClear(*data); } } @@ -954,7 +954,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi if (info->unitNum >= info->unitSize) { uint32_t psize = info->unitSize; info->unitSize += FILTER_DEFAULT_UNIT_SIZE; - info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); + info->units = taosMemoryRealloc(info->units, info->unitSize * sizeof(SFilterUnit)); memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); } @@ -1000,7 +1000,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) { if (group->unitNum >= group->unitSize) { group->unitSize += FILTER_DEFAULT_UNIT_SIZE; - group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); + group->unitIdxs = taosMemoryRealloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); } group->unitIdxs[group->unitNum++] = unitIdx; @@ -1028,12 +1028,12 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { in.type = valueNode->node.resType.type; in.bytes = valueNode->node.resType.bytes; in.data = nodesGetValueFromNode(valueNode); - out.data = malloc(sizeof(int64_t)); + out.data = taosMemoryMalloc(sizeof(int64_t)); code = vectorConvertImpl(&in, &out); if (code) { fltError("convert from %d to %d failed", in.type, out.type); - tfree(out.data); + taosMemoryFreeClear(out.data); FLT_ERR_RET(code); } @@ -1148,17 +1148,17 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); if (func(&ra->s, &ra->e) == 0) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } else { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); + void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &ra->e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); @@ -1170,7 +1170,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); @@ -1178,7 +1178,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); @@ -1224,16 +1224,16 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if ((!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) &&(!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL))) { __compar_fn_t func = getComparFunc(type, 0); if (func(&r->ra.s, &r->ra.e) == 0) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } else { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - void *data2 = malloc(sizeof(int64_t)); + void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &r->ra.e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); @@ -1250,7 +1250,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); @@ -1258,7 +1258,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan } if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { - void *data = malloc(sizeof(int64_t)); + void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); @@ -1284,8 +1284,8 @@ static void filterFreeGroup(void *pItem) { } SFilterGroup* p = (SFilterGroup*) pItem; - tfree(p->unitIdxs); - tfree(p->unitFlags); + taosMemoryFreeClear(p->unitIdxs); + taosMemoryFreeClear(p->unitFlags); } @@ -1635,7 +1635,7 @@ void filterFreeGroupCtx(SFilterGroupCtx* gRes) { return; } - tfree(gRes->colIdx); + taosMemoryFreeClear(gRes->colIdx); int16_t i = 0, j = 0; @@ -1648,8 +1648,8 @@ void filterFreeGroupCtx(SFilterGroupCtx* gRes) { ++j; } - tfree(gRes->colInfo); - tfree(gRes); + taosMemoryFreeClear(gRes->colInfo); + taosMemoryFreeClear(gRes); } void filterFreeField(SFilterField* field, int32_t type) { @@ -1661,7 +1661,7 @@ void filterFreeField(SFilterField* field, int32_t type) { if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { taosHashCleanup(field->data); } else { - tfree(field->data); + taosMemoryFreeClear(field->data); } } } @@ -1676,40 +1676,40 @@ void filterFreeInfo(SFilterInfo *info) { return; } - tfree(info->cunits); - tfree(info->blkUnitRes); - tfree(info->blkUnits); + taosMemoryFreeClear(info->cunits); + taosMemoryFreeClear(info->blkUnitRes); + taosMemoryFreeClear(info->blkUnits); for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { for (uint32_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); } - tfree(info->fields[i].fields); + taosMemoryFreeClear(info->fields[i].fields); } for (uint32_t i = 0; i < info->groupNum; ++i) { filterFreeGroup(&info->groups[i]); } - tfree(info->groups); + taosMemoryFreeClear(info->groups); - tfree(info->units); + taosMemoryFreeClear(info->units); - tfree(info->unitRes); + taosMemoryFreeClear(info->unitRes); - tfree(info->unitFlags); + taosMemoryFreeClear(info->unitFlags); for (uint32_t i = 0; i < info->colRangeNum; ++i) { filterFreeRangeCtx(info->colRange[i]); } - tfree(info->colRange); + taosMemoryFreeClear(info->colRange); filterFreePCtx(&info->pctx); if (!FILTER_GET_FLAG(info->status, FI_STATUS_CLONED)) { - tfree(info); + taosMemoryFreeClear(info); } } @@ -1775,14 +1775,14 @@ int32_t fltInitValFieldData(SFilterInfo *info) { if (type == TSDB_DATA_TYPE_BINARY) { size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; - fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); + fi->data = taosMemoryCalloc(1, len + 1 + VARSTR_HEADER_SIZE); } else if (type == TSDB_DATA_TYPE_NCHAR) { size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; - fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + fi->data = taosMemoryCalloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); } else if (type != TSDB_DATA_TYPE_JSON){ if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE /* - fi->data = calloc(dType->bytes, tDataTypes[type].bytes); + fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes); for (int32_t a = 0; a < dType->bytes; ++a) { int64_t *v = taosArrayGet(var->arr, a); assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); @@ -1790,7 +1790,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { */ continue; } else { - fi->data = calloc(1, sizeof(int64_t)); + fi->data = taosMemoryCalloc(1, sizeof(int64_t)); } } else{ // type == TSDB_DATA_TYPE_JSON // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc) @@ -1998,15 +1998,15 @@ _return: int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { bool empty = false; - uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t)); + uint32_t *colIdx = taosMemoryMalloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t)); uint32_t colIdxi = 0; uint32_t gResIdx = 0; for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup* g = info->groups + i; - gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); - gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); + gRes[gResIdx] = taosMemoryCalloc(1, sizeof(SFilterGroupCtx)); + gRes[gResIdx]->colInfo = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); colIdxi = 0; empty = false; @@ -2058,7 +2058,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t ++gResIdx; } - tfree(colIdx); + taosMemoryFreeClear(colIdx); *gResNum = gResIdx; @@ -2350,12 +2350,12 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { info->groupNum = (uint32_t)groupSize; if (info->groupNum > 0) { - info->groups = calloc(info->groupNum, sizeof(*info->groups)); + info->groups = taosMemoryCalloc(info->groupNum, sizeof(*info->groups)); } for (size_t i = 0; i < groupSize; ++i) { SFilterGroup *pg = taosArrayGet(group, i); - pg->unitFlags = calloc(pg->unitNum, sizeof(*pg->unitFlags)); + pg->unitFlags = taosMemoryCalloc(pg->unitNum, sizeof(*pg->unitFlags)); info->groups[i] = *pg; } @@ -2435,7 +2435,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ uint32_t *idxs = NULL; uint32_t colNum = 0; SFilterGroupCtx *res = NULL; - uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); + uint32_t *idxNum = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); for (int32_t i = 0; i < gResNum; ++i) { for (uint32_t m = 0; m < gRes[i]->colNum; ++m) { @@ -2456,7 +2456,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ assert(idxNum[i] == gResNum); if (idxs == NULL) { - idxs = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); + idxs = taosMemoryCalloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); } idxs[colNum++] = i; @@ -2465,7 +2465,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ FLT_CHK_JMP(colNum <= 0); info->colRangeNum = colNum; - info->colRange = calloc(colNum, POINTER_BYTES); + info->colRange = taosMemoryCalloc(colNum, POINTER_BYTES); for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; @@ -2512,8 +2512,8 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ } _return: - tfree(idxNum); - tfree(idxs); + taosMemoryFreeClear(idxNum); + taosMemoryFreeClear(idxs); return TSDB_CODE_SUCCESS; } @@ -2533,9 +2533,9 @@ int32_t filterPostProcessRange(SFilterInfo *info) { int32_t filterGenerateComInfo(SFilterInfo *info) { - info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); - info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); - info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); + info->cunits = taosMemoryMalloc(info->unitNum * sizeof(*info->cunits)); + info->blkUnitRes = taosMemoryMalloc(sizeof(*info->blkUnitRes) * info->unitNum); + info->blkUnits = taosMemoryMalloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; @@ -2766,7 +2766,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint32_t *unitIdx = NULL; if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2871,7 +2871,7 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2905,7 +2905,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2947,7 +2947,7 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2977,7 +2977,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -2991,7 +2991,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa // match/nmatch for nchar type need convert from ucs4 to mbs if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){ - char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + char *newColData = taosMemoryCalloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ qError("castConvert1 taosUcs4ToMbs error"); @@ -2999,7 +2999,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa varDataSetLen(newColData, len); (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); } - tfree(newColData); + taosMemoryFreeClear(newColData); }else{ (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); } @@ -3022,7 +3022,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg } if (*p == NULL) { - *p = calloc(numOfRows, sizeof(int8_t)); + *p = taosMemoryCalloc(numOfRows, sizeof(int8_t)); } for (int32_t i = 0; i < numOfRows; ++i) { @@ -3051,7 +3051,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){ - char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + char *newColData = taosMemoryCalloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ qError("castConvert1 taosUcs4ToMbs error"); @@ -3059,7 +3059,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg varDataSetLen(newColData, len); (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); } - tfree(newColData); + taosMemoryFreeClear(newColData); }else{ (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); } @@ -3125,7 +3125,7 @@ int32_t filterSetExecFunc(SFilterInfo *info) { int32_t filterPreprocess(SFilterInfo *info) { - SFilterGroupCtx** gRes = calloc(info->groupNum, sizeof(SFilterGroupCtx *)); + SFilterGroupCtx** gRes = taosMemoryCalloc(info->groupNum, sizeof(SFilterGroupCtx *)); int32_t gResNum = 0; filterMergeGroupUnits(info, gRes, &gResNum); @@ -3161,7 +3161,7 @@ _return: filterFreeGroupCtx(gRes[i]); } - tfree(gRes); + taosMemoryFreeClear(gRes); return TSDB_CODE_SUCCESS; } @@ -3216,8 +3216,8 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { } } - info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); - info->unitFlags = malloc(info->unitNum * sizeof(*info->unitFlags)); + info->unitRes = taosMemoryMalloc(info->unitNum * sizeof(*info->unitRes)); + info->unitFlags = taosMemoryMalloc(info->unitNum * sizeof(*info->unitFlags)); filterDumpInfoToString(info, "Final", 0); @@ -3418,7 +3418,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar SFilterField nfi = {0}; nfi.desc = fi->desc; int32_t bytes = FILTER_GET_COL_FIELD_SIZE(fi); - nfi.data = malloc(rows * bytes); + nfi.data = taosMemoryMalloc(rows * bytes); int32_t bufSize = bytes - VARSTR_HEADER_SIZE; for (int32_t j = 0; j < rows; ++j) { char *src = FILTER_GET_COL_FIELD_DATA(fi, j); @@ -3459,7 +3459,7 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); if (type == TSDB_DATA_TYPE_NCHAR) { - tfree(fi->data); + taosMemoryFreeClear(fi->data); } } @@ -3634,7 +3634,7 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) } if (*pInfo == NULL) { - *pInfo = calloc(1, sizeof(SFilterInfo)); + *pInfo = taosMemoryCalloc(1, sizeof(SFilterInfo)); if (NULL == *pInfo) { fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo)); FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 8e7eaa0f8c..83902c9df8 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -31,7 +31,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; int8_t dummy = 0; int32_t bufLen = 60; - out.data = malloc(bufLen); + out.data = taosMemoryMalloc(bufLen); int32_t len = 0; void *buf = NULL; @@ -75,14 +75,14 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { cell = cell->pNext; } - tfree(out.data); + taosMemoryFreeClear(out.data); *data = pObj; return TSDB_CODE_SUCCESS; _return: - tfree(out.data); + taosMemoryFreeClear(out.data); taosHashCleanup(pObj); SCL_RET(code); @@ -98,7 +98,7 @@ FORCE_INLINE bool sclIsNull(SScalarParam* param, int32_t idx) { FORCE_INLINE void sclSetNull(SScalarParam* param, int32_t idx) { if (NULL == param->bitmap) { - param->bitmap = calloc(BitmapLen(param->num), sizeof(char)); + param->bitmap = taosMemoryCalloc(BitmapLen(param->num), sizeof(char)); if (NULL == param->bitmap) { sclError("calloc %d failed", param->num); return; @@ -126,7 +126,7 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParamNoData(SScalarParam *param) { - tfree(param->bitmap); + taosMemoryFreeClear(param->bitmap); } @@ -137,7 +137,7 @@ void sclFreeParam(SScalarParam *param) { if (SCL_DATA_TYPE_DUMMY_HASH == param->type) { taosHashCleanup((SHashObj *)param->orig.data); } else { - tfree(param->orig.data); + taosMemoryFreeClear(param->orig.data); } } } @@ -148,7 +148,7 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { return TSDB_CODE_SUCCESS; } - *res = malloc(pNode->node.resType.bytes); + *res = taosMemoryMalloc(pNode->node.resType.bytes); if (NULL == (*res)) { sclError("malloc %d failed", pNode->node.resType.bytes); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -284,7 +284,7 @@ int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx) int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { int32_t code = 0; - SScalarParam *paramList = calloc(pParamList->length, sizeof(SScalarParam)); + SScalarParam *paramList = taosMemoryCalloc(pParamList->length, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -308,7 +308,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarC _return: - tfree(paramList); + taosMemoryFreeClear(paramList); SCL_RET(code); } @@ -320,7 +320,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SScalarParam *paramList = calloc(paramNum, sizeof(SScalarParam)); + SScalarParam *paramList = taosMemoryCalloc(paramNum, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -337,7 +337,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal _return: - tfree(paramList); + taosMemoryFreeClear(paramList); SCL_RET(code); } @@ -360,7 +360,7 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); output->type = node->node.resType.type; - output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes)); + output->data = taosMemoryCalloc(rowNum, sizeof(tDataTypes[output->type].bytes)); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -384,7 +384,7 @@ _return: sclFreeParamNoData(params + i); } - tfree(params); + taosMemoryFreeClear(params); SCL_RET(code); } @@ -415,7 +415,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->type = node->node.resType.type; output->bytes = sizeof(bool); output->num = rowNum; - output->data = calloc(rowNum, sizeof(bool)); + output->data = taosMemoryCalloc(rowNum, sizeof(bool)); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -449,7 +449,7 @@ _return: sclFreeParamNoData(params + i); } - tfree(params); + taosMemoryFreeClear(params); SCL_RET(code); } @@ -463,7 +463,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp output->type = node->node.resType.type; output->num = rowNum; output->bytes = tDataTypes[output->type].bytes; - output->data = calloc(rowNum, tDataTypes[output->type].bytes); + output->data = taosMemoryCalloc(rowNum, tDataTypes[output->type].bytes); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -485,7 +485,7 @@ _return: sclFreeParamNoData(params + i); } - tfree(params); + taosMemoryFreeClear(params); SCL_RET(code); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index fd900afcda..f2fdb29e4a 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -4,145 +4,536 @@ static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) { dst->type = src->type; dst->bytes = src->bytes; - dst->num = src->num; + //dst->num = src->num; } -static void tceil(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assignBasicParaInfo(pOutput, pLeft); - assert(numOfInput == 1); - - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = ceilf(p[i]); - } - } - - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*)pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = ceil(p[i]); - } - } - - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); +/** Math functions **/ +int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + assignBasicParaInfo(pOutput, pInput); + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; } + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + switch (pInput->type) { + case TSDB_DATA_TYPE_FLOAT: { + float v; + GET_TYPED_DATA(v, float, pInput->type, input); + float result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_TINYINT: { + int8_t v; + GET_TYPED_DATA(v, int8_t, pInput->type, input); + int8_t result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { + int16_t v; + GET_TYPED_DATA(v, int16_t, pInput->type, input); + int16_t result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_INT: { + int32_t v; + GET_TYPED_DATA(v, int32_t, pInput->type, input); + int32_t result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + int64_t v; + GET_TYPED_DATA(v, int64_t, pInput->type, input); + int64_t result; + result = (v > 0) ? v : -v; + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + default: { + memcpy(output, input, pInput->bytes); + break; + } + } + } + + return TSDB_CODE_SUCCESS; } -static void tfloor(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assignBasicParaInfo(pOutput, pLeft); - assert(numOfInput == 1); - - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = floorf(p[i]); - } - } - - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*) pOutput->data; - - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = floor(p[i]); - } - } - - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); +int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 2 || !IS_NUMERIC_TYPE(pInput[0].type) || !IS_NUMERIC_TYPE(pInput[1].type)) { + return TSDB_CODE_FAILED; } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char **input = NULL, *output = NULL; + bool hasNullInput = false; + input = taosMemoryCalloc(inputNum, sizeof(char *)); + for (int32_t i = 0; i < pOutput->num; ++i) { + for (int32_t j = 0; j < inputNum; ++j) { + if (pInput[j].num == 1) { + input[j] = pInput[j].data; + } else { + input[j] = pInput[j].data + i * pInput[j].bytes; + } + if (isNull(input[j], pInput[j].type)) { + hasNullInput = true; + break; + } + } + output = pOutput->data + i * pOutput->bytes; + + if (hasNullInput) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double base; + GET_TYPED_DATA(base, double, pInput[1].type, input[1]); + double v; + GET_TYPED_DATA(v, double, pInput[0].type, input[0]); + double result = log(v) / log(base); + SET_TYPED_DATA(output, pOutput->type, result); + } + + taosMemoryFree(input); + + return TSDB_CODE_SUCCESS; } -static void _tabs(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assignBasicParaInfo(pOutput, pLeft); - assert(numOfInput == 1); - - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t* p = (int8_t*) pLeft->data; - int8_t* out = (int8_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_SMALLINT: { - int16_t* p = (int16_t*) pLeft->data; - int16_t* out = (int16_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_INT: { - int32_t* p = (int32_t*) pLeft->data; - int32_t* out = (int32_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_BIGINT: { - int64_t* p = (int64_t*) pLeft->data; - int64_t* out = (int64_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); +int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 2 || !IS_NUMERIC_TYPE(pInput[0].type) || !IS_NUMERIC_TYPE(pInput[1].type)) { + return TSDB_CODE_FAILED; } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char **input = NULL, *output = NULL; + bool hasNullInput = false; + input = taosMemoryCalloc(inputNum, sizeof(char *)); + for (int32_t i = 0; i < pOutput->num; ++i) { + for (int32_t j = 0; j < inputNum; ++j) { + if (pInput[j].num == 1) { + input[j] = pInput[j].data; + } else { + input[j] = pInput[j].data + i * pInput[j].bytes; + } + if (isNull(input[j], pInput[j].type)) { + hasNullInput = true; + break; + } + } + output = pOutput->data + i * pOutput->bytes; + + if (hasNullInput) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double base; + GET_TYPED_DATA(base, double, pInput[1].type, input[1]); + double v; + GET_TYPED_DATA(v, double, pInput[0].type, input[0]); + double result = pow(v, base); + SET_TYPED_DATA(output, pOutput->type, result); + } + + taosMemoryFree(input); + + return TSDB_CODE_SUCCESS; } -static void tround(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assignBasicParaInfo(pOutput, pLeft); - assert(numOfInput == 1); - - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = roundf(p[i]); - } - } - - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = round(p[i]); - } - } - - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); +int32_t sqrtFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = sqrt(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t sinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = sin(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t cosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = cos(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = tan(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t asinFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = asin(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t acosFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = acos(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + pOutput->type = TSDB_DATA_TYPE_DOUBLE; + pOutput->bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = atan(v); + SET_TYPED_DATA(output, pOutput->type, result); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + switch (pInput->type) { + case TSDB_DATA_TYPE_FLOAT: { + float v; + GET_TYPED_DATA(v, float, pInput->type, input); + float result = ceilf(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = ceil(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + default: { + memcpy(output, input, pInput->bytes); + break; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + assignBasicParaInfo(pOutput, pInput); + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + switch (pInput->type) { + case TSDB_DATA_TYPE_FLOAT: { + float v; + GET_TYPED_DATA(v, float, pInput->type, input); + float result = floorf(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = floor(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + default: { + memcpy(output, input, pInput->bytes); + break; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + assignBasicParaInfo(pOutput, pInput); + if (inputNum != 1 || !IS_NUMERIC_TYPE(pInput->type)) { + return TSDB_CODE_FAILED; + } + + char *input = NULL, *output = NULL; + for (int32_t i = 0; i < pOutput->num; ++i) { + if (pInput->num == 1) { + input = pInput->data; + } else { + input = pInput->data + i * pInput->bytes; + } + output = pOutput->data + i * pOutput->bytes; + + if (isNull(input, pInput->type)) { + setNull(output, pOutput->type, pOutput->bytes); + continue; + } + + switch (pInput->type) { + case TSDB_DATA_TYPE_FLOAT: { + float v; + GET_TYPED_DATA(v, float, pInput->type, input); + float result = roundf(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double v; + GET_TYPED_DATA(v, double, pInput->type, input); + double result = round(v); + SET_TYPED_DATA(output, pOutput->type, result); + break; + } + + default: { + memcpy(output, input, pInput->bytes); + break; + } + } + } + + return TSDB_CODE_SUCCESS; } static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { @@ -169,7 +560,7 @@ static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam } } - pOutput->data = realloc(pOutput->data, rowLen * num); + pOutput->data = taosMemoryRealloc(pOutput->data, rowLen * num); assert(pOutput->data); char* rstart = pOutput->data; @@ -284,13 +675,13 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa SScalarFuncParam rightOutput = {0}; if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) { - leftOutput.data = malloc(sizeof(int64_t) * numOfRows); + leftOutput.data = taosMemoryMalloc(sizeof(int64_t) * numOfRows); evaluateExprNodeTree(pLeft, numOfRows, &leftOutput, param, getSourceDataBlock); } // the right output has result from the right child syntax tree if (pRight->nodeType == TEXPR_BINARYEXPR_NODE || pRight->nodeType == TEXPR_UNARYEXPR_NODE) { - rightOutput.data = malloc(sizeof(int64_t) * numOfRows); + rightOutput.data = taosMemoryMalloc(sizeof(int64_t) * numOfRows); evaluateExprNodeTree(pRight, numOfRows, &rightOutput, param, getSourceDataBlock); } @@ -322,7 +713,7 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa void* outputBuf = pOutput->data; if (isStringOp(pExprs->_node.optr)) { - outputBuf = realloc(pOutput->data, (left.bytes + right.bytes) * left.num); + outputBuf = taosMemoryRealloc(pOutput->data, (left.bytes + right.bytes) * left.num); } OperatorFn(&left, &right, outputBuf, TSDB_ORDER_ASC); @@ -345,7 +736,7 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa // reserve enough memory buffer if (isBinaryStringOp(pExprs->_node.optr)) { - void* outputBuf = realloc(pOutput->data, left.bytes * left.num); + void* outputBuf = taosMemoryRealloc(pOutput->data, left.bytes * left.num); assert(outputBuf != NULL); pOutput->data = outputBuf; } @@ -353,24 +744,24 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa OperatorFn(&left, pOutput); } - tfree(leftOutput.data); - tfree(rightOutput.data); + taosMemoryFreeClear(leftOutput.data); + taosMemoryFreeClear(rightOutput.data); return 0; } #endif -SScalarFunctionInfo scalarFunc[8] = { - {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil}, - {"floor", FUNCTION_TYPE_SCALAR, FUNCTION_FLOOR, tfloor}, - {"abs", FUNCTION_TYPE_SCALAR, FUNCTION_ABS, _tabs}, - {"round", FUNCTION_TYPE_SCALAR, FUNCTION_ROUND, tround}, - {"length", FUNCTION_TYPE_SCALAR, FUNCTION_LENGTH, tlength}, - {"concat", FUNCTION_TYPE_SCALAR, FUNCTION_CONCAT, tconcat}, - {"ltrim", FUNCTION_TYPE_SCALAR, FUNCTION_LTRIM, tltrim}, - {"rtrim", FUNCTION_TYPE_SCALAR, FUNCTION_RTRIM, trtrim}, -}; +//SScalarFunctionInfo scalarFunc[8] = { +// {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil}, +// {"floor", FUNCTION_TYPE_SCALAR, FUNCTION_FLOOR, tfloor}, +// {"abs", FUNCTION_TYPE_SCALAR, FUNCTION_ABS, _tabs}, +// {"round", FUNCTION_TYPE_SCALAR, FUNCTION_ROUND, tround}, +// {"length", FUNCTION_TYPE_SCALAR, FUNCTION_LENGTH, tlength}, +// {"concat", FUNCTION_TYPE_SCALAR, FUNCTION_CONCAT, tconcat}, +// {"ltrim", FUNCTION_TYPE_SCALAR, FUNCTION_LTRIM, tltrim}, +// {"rtrim", FUNCTION_TYPE_SCALAR, FUNCTION_RTRIM, trtrim}, +//}; void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols; @@ -379,13 +770,13 @@ void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprI return; } - sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); + sas->colList = taosMemoryCalloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); for(int32_t i = 0; i < sas->numOfCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i); sas->colList[i] = pColData->info; } - sas->data = calloc(sas->numOfCols, POINTER_BYTES); + sas->data = taosMemoryCalloc(sas->numOfCols, POINTER_BYTES); // set the input column data for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) { @@ -395,7 +786,7 @@ void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprI } SScalarFunctionSupport* createScalarFuncSupport(int32_t num) { - SScalarFunctionSupport* pSupp = calloc(num, sizeof(SScalarFunctionSupport)); + SScalarFunctionSupport* pSupp = taosMemoryCalloc(num, sizeof(SScalarFunctionSupport)); return pSupp; } @@ -406,9 +797,9 @@ void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t n for(int32_t i = 0; i < num; ++i) { SScalarFunctionSupport* pSupp = &pSupport[i]; - tfree(pSupp->data); - tfree(pSupp->colList); + taosMemoryFreeClear(pSupp->data); + taosMemoryFreeClear(pSupp->colList); } - tfree(pSupport); -} \ No newline at end of file + taosMemoryFreeClear(pSupport); +} diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 17ac9b19fd..2300f5a1d3 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -305,7 +305,7 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t if (TSDB_DATA_TYPE_BINARY == inType) { if (varDataLen(pIn->data) >= bufSize) { bufSize = varDataLen(pIn->data) + 1; - tmp = realloc(tmp, bufSize); + tmp = taosMemoryRealloc(tmp, bufSize); } memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); @@ -313,13 +313,13 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t } else { if (varDataLen(pIn->data) * TSDB_NCHAR_SIZE >= bufSize) { bufSize = varDataLen(pIn->data) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); + tmp = taosMemoryRealloc(tmp, bufSize); } int len = taosUcs4ToMbs((TdUcs4*)varDataVal(pIn->data), varDataLen(pIn->data), tmp); if (len < 0){ sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); + taosMemoryFreeClear(tmp); return TSDB_CODE_QRY_APP_ERROR; } @@ -329,7 +329,7 @@ int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t (*func)(tmp, pOut, outType); } - tfree(tmp); + taosMemoryFreeClear(tmp); return TSDB_CODE_SUCCESS; } @@ -480,7 +480,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p paramOut1->bytes = param1->bytes; paramOut1->type = type; paramOut1->num = param1->num; - paramOut1->data = malloc(paramOut1->num * tDataTypes[paramOut1->type].bytes); + paramOut1->data = taosMemoryMalloc(paramOut1->num * tDataTypes[paramOut1->type].bytes); if (NULL == paramOut1->data) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -488,7 +488,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p code = vectorConvertImpl(param1, paramOut1); if (code) { - tfree(paramOut1->data); + taosMemoryFreeClear(paramOut1->data); return code; } } @@ -497,17 +497,17 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p paramOut2->bytes = param2->bytes; paramOut2->type = type; paramOut2->num = param2->num; - paramOut2->data = malloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); + paramOut2->data = taosMemoryMalloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); if (NULL == paramOut2->data) { - tfree(paramOut1->data); + taosMemoryFreeClear(paramOut1->data); return TSDB_CODE_QRY_OUT_OF_MEMORY; } paramOut2->orig.data = paramOut2->data; code = vectorConvertImpl(param2, paramOut2); if (code) { - tfree(paramOut1->data); - tfree(paramOut2->data); + taosMemoryFreeClear(paramOut1->data); + taosMemoryFreeClear(paramOut2->data); return code; } } @@ -523,7 +523,7 @@ void vectorMath(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, i SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -536,7 +536,7 @@ void vectorMath(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, i pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -681,7 +681,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -694,7 +694,7 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -774,7 +774,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -787,7 +787,7 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, in pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -863,7 +863,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -876,7 +876,7 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -953,7 +953,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -966,7 +966,7 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -1040,7 +1040,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(double)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -1053,7 +1053,7 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(double)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -1206,7 +1206,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -1219,7 +1219,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); @@ -1303,7 +1303,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { - leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + leftParam.data = taosMemoryCalloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; @@ -1316,7 +1316,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pLeft = &leftParam; } if (IS_VAR_DATA_TYPE(pRight->type)) { - rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + rightParam.data = taosMemoryCalloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); sclFreeParam(&leftParam); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 328028a1d4..54f82eae2d 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -71,7 +71,7 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { vnode->node.resType.type = dataType; if (IS_VAR_DATA_TYPE(dataType)) { - vnode->datum.p = (char *)malloc(varDataTLen(value)); + vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); varDataCopy(vnode->datum.p, value); vnode->node.resType.bytes = varDataLen(value); } else { @@ -102,7 +102,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 3; res->info.rows = rowNum; res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); @@ -113,7 +113,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); } @@ -122,7 +122,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 3; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); blockDataEnsureCapacity(res, rowNum); @@ -150,7 +150,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 1 + idx; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -325,7 +325,7 @@ TEST(columnTest, smallint_column_greater_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); blockDataDestroy(src); nodesDestroyNode(opNode); @@ -380,7 +380,7 @@ TEST(columnTest, int_column_greater_smallint_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -426,7 +426,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -491,7 +491,7 @@ TEST(columnTest, binary_column_in_binary_list) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -540,7 +540,7 @@ TEST(columnTest, binary_column_like_binary) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -588,7 +588,7 @@ TEST(columnTest, binary_column_is_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -635,7 +635,7 @@ TEST(columnTest, binary_column_is_not_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -674,7 +674,7 @@ TEST(opTest, smallint_column_greater_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -713,7 +713,7 @@ TEST(opTest, smallint_value_add_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -758,7 +758,7 @@ TEST(opTest, bigint_column_multi_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -801,7 +801,7 @@ TEST(opTest, smallint_column_and_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -839,7 +839,7 @@ TEST(opTest, smallint_column_or_float_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -879,7 +879,7 @@ TEST(opTest, smallint_column_or_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -924,7 +924,7 @@ TEST(opTest, binary_column_is_true) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(opNode); blockDataDestroy(src); @@ -996,7 +996,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1065,7 +1065,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1135,7 +1135,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1204,7 +1204,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); @@ -1277,7 +1277,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - tfree(rowRes); + taosMemoryFreeClear(rowRes); filterFreeInfo(filter); nodesDestroyNode(logicNode1); blockDataDestroy(src); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index d6a73d99bb..f0025de6b8 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -39,6 +39,14 @@ #include "nodes.h" #include "tlog.h" +#define _DEBUG_PRINT_ 0 + +#if _DEBUG_PRINT_ +#define PRINTF(...) printf(__VA_ARGS__) +#else +#define PRINTF(...) +#endif + namespace { SColumnInfo createColumnInfo(int32_t colId, int32_t type, int32_t bytes) { @@ -71,7 +79,7 @@ void scltInitLogFile() { void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { if (newBlock) { - SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 1; res->info.rows = rows; res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); @@ -106,7 +114,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { vnode->node.resType.type = dataType; if (IS_VAR_DATA_TYPE(dataType)) { - vnode->datum.p = (char *)malloc(varDataTLen(value)); + vnode->datum.p = (char *)taosMemoryMalloc(varDataTLen(value)); varDataCopy(vnode->datum.p, value); vnode->node.resType.bytes = varDataTLen(value); } else { @@ -125,7 +133,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->dataBlockId = 0; if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 3; res->info.rows = rowNum; res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); @@ -136,7 +144,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); } @@ -145,7 +153,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 3; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); blockDataEnsureCapacity(res, rowNum); @@ -173,7 +181,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.bytes = dataBytes; idata.info.colId = 1 + idx; int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)calloc(1, size); + idata.pData = (char *)taosMemoryCalloc(1, size); taosArrayPush(res->pDataBlock, &idata); res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -1433,6 +1441,1645 @@ TEST(columnTest, greater_and_lower) { nodesDestroyNode(logicNode); } +void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t num, bool setVal) { + SScalarParam *input = (SScalarParam *)taosMemoryCalloc(1, sizeof(SScalarParam)); + int32_t bytes; + switch (type) { + case TSDB_DATA_TYPE_TINYINT: { + bytes = sizeof(int8_t); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + bytes = sizeof(int16_t); + break; + } + case TSDB_DATA_TYPE_INT: { + bytes = sizeof(int32_t); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + bytes = sizeof(int64_t); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + bytes = sizeof(float); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + bytes = sizeof(double); + break; + } + } + + input->type = type; + input->num = num; + input->data = taosMemoryCalloc(num, bytes); + input->bytes = bytes; + if (setVal) { + for (int32_t i = 0; i < num; ++i) { + memcpy(input->data + i * bytes, pVal, bytes); + } + } else { + memset(input->data, 0, num * bytes); + } + + *pInput = input; +} + +void scltDestroyDataBlock(SScalarParam *pInput) { + taosMemoryFree(pInput->data); + taosMemoryFree(pInput); +} + +TEST(ScalarFunctionTest, absFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), val_tinyint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_tinyint = -10; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), -val_tinyint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //SMALLINT + int16_t val_smallint = 10; + type = TSDB_DATA_TYPE_SMALLINT; + scltMakeDataBlock(&pInput, type, &val_smallint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int16_t *)pOutput->data + i), val_smallint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_smallint = -10; + scltMakeDataBlock(&pInput, type, &val_smallint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int16_t *)pOutput->data + i), -val_smallint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //INT + int32_t val_int = 10; + type = TSDB_DATA_TYPE_INT; + scltMakeDataBlock(&pInput, type, &val_int, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int32_t *)pOutput->data + i), val_int); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_int = -10; + scltMakeDataBlock(&pInput, type, &val_int, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int32_t *)pOutput->data + i), -val_int); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //BIGINT + int64_t val_bigint = 10; + type = TSDB_DATA_TYPE_BIGINT; + scltMakeDataBlock(&pInput, type, &val_bigint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)pOutput->data + i), val_bigint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_bigint = -10; + scltMakeDataBlock(&pInput, type, &val_bigint, rowNum, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)pOutput->data + i), -val_bigint); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 10.15; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before ABS:%f\n", *(float *)pInput->data); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), val_float); + PRINTF("float after ABS:%f\n", *((float *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_float = -10.15; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before ABS:%f\n", *(float *)pInput->data); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), -val_float); + PRINTF("float after ABS:%f\n", *((float *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //DOUBLE + double val_double = 10.15; + type = TSDB_DATA_TYPE_DOUBLE; + scltMakeDataBlock(&pInput, type, &val_double, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), val_double); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_double = -10.15; + scltMakeDataBlock(&pInput, type, &val_double, rowNum, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), -val_double); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + +} + +TEST(ScalarFunctionTest, absFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 5; + int32_t type; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint + i; + PRINTF("tiny_int before ABS:%d\n", *((int8_t *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), val_tinyint + i); + PRINTF("tiny_int after ABS:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_tinyint = -10; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint + i; + PRINTF("tiny_int before ABS:%d\n", *((int8_t *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), -(val_tinyint + i)); + PRINTF("tiny_int after ABS:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //SMALLINT + int16_t val_smallint = 10; + type = TSDB_DATA_TYPE_SMALLINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int16_t *)pInput->data + i) = val_smallint + i; + PRINTF("small_int before ABS:%d\n", *((int16_t *)pInput->data + i)); + } + + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int16_t *)pOutput->data + i), val_smallint + i); + PRINTF("small_int after ABS:%d\n", *((int16_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_smallint = -10; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int16_t *)pInput->data + i) = val_smallint + i; + PRINTF("small_int before ABS:%d\n", *((int16_t *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int16_t *)pOutput->data + i), -(val_smallint + i)); + PRINTF("small_int after ABS:%d\n", *((int16_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //INT + int32_t val_int = 10; + type = TSDB_DATA_TYPE_INT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int32_t *)pInput->data + i) = val_int + i; + PRINTF("int before ABS:%d\n", *((int32_t *)pInput->data + i)); + } + + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int32_t *)pOutput->data + i), val_int + i); + PRINTF("int after ABS:%d\n", *((int32_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_int = -10; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int32_t *)pInput->data + i) = val_int + i; + PRINTF("int before ABS:%d\n", *((int32_t *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int32_t *)pOutput->data + i), -(val_int + i)); + PRINTF("int after ABS:%d\n", *((int32_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 10.15; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float + i; + PRINTF("float before ABS:%f\n", *((float *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), val_float + i); + PRINTF("float after ABS:%f\n", *((float *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_float = -10.15; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float + i; + PRINTF("float before ABS:%f\n", *((float *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), -(val_float + i)); + PRINTF("float after ABS:%f\n", *((float *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //DOUBLE + double val_double = 10.15; + type = TSDB_DATA_TYPE_DOUBLE; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((double *)pInput->data + i) = val_double + i; + PRINTF("double before ABS:%f\n", *((double *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), val_double + i); + PRINTF("double after ABS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + val_double = -10.15; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((double *)pInput->data + i) = val_double + i; + PRINTF("double before ABS:%f\n", *((double *)pInput->data + i)); + } + + code = absFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), -(val_double + i)); + PRINTF("double after ABS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, sinFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.42016703682664092; + + //TINYINT + int8_t val_tinyint = 13; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before SIN:%d\n", *((int8_t *)pInput->data)); + + code = sinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after SIN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 13.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before SIN:%f\n", *((float *)pInput->data)); + + code = sinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after SIN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + +} + +TEST(ScalarFunctionTest, sinFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {0.42016703682664092, 0.99060735569487035, 0.65028784015711683}; + + + //TINYINT + int8_t val_tinyint[] = {13, 14, 15}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before SIN:%d\n", *((int8_t *)pInput->data + i)); + } + + code = sinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after SIN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {13.00, 14.00, 15.00}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before SIN:%f\n", *((float *)pInput->data + i)); + } + + code = sinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after SIN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, cosFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.90744678145019619; + + //TINYINT + int8_t val_tinyint = 13; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before COS:%d\n", *((int8_t *)pInput->data)); + + code = cosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after COS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 13.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before COS:%f\n", *((float *)pInput->data)); + + code = cosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after COS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, cosFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {0.90744678145019619, 0.13673721820783361, -0.75968791285882131}; + + //TINYINT + int8_t val_tinyint[] = {13, 14, 15}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before COS:%d\n", *((int8_t *)pInput->data + i)); + } + + code = cosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after COS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {13.00, 14.00, 15.00}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before COS:%f\n", *((float *)pInput->data + i)); + } + + code = cosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after COS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, tanFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.46302113293648961; + + //TINYINT + int8_t val_tinyint = 13; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before TAN:%d\n", *((int8_t *)pInput->data)); + + code = tanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after TAN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 13.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before TAN:%f\n", *((float *)pInput->data)); + + code = tanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after TAN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, tanFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {0.46302113293648961, 7.24460661609480550, -0.85599340090851872}; + + //TINYINT + int8_t val_tinyint[] = {13, 14, 15}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before TAN:%d\n", *((int8_t *)pInput->data + i)); + } + + code = tanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after TAN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {13.00, 14.00, 15.00}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before TAN:%f\n", *((float *)pInput->data + i)); + } + + code = tanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after TAN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, asinFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 1.57079632679489656; + + //TINYINT + int8_t val_tinyint = 1; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before ASIN:%d\n", *((int8_t *)pInput->data)); + + code = asinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after ASIN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 1.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before ASIN:%f\n", *((float *)pInput->data)); + + code = asinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after ASIN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, asinFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {-1.57079632679489656, 0.0, 1.57079632679489656}; + + + //TINYINT + int8_t val_tinyint[] = {-1, 0, 1}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before ASIN:%d\n", *((int8_t *)pInput->data + i)); + } + + code = asinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after ASIN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-1.0, 0.0, 1.0}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before ASIN:%f\n", *((float *)pInput->data + i)); + } + + code = asinFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after ASIN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, acosFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.0; + + //TINYINT + int8_t val_tinyint = 1; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before ACOS:%d\n", *((int8_t *)pInput->data)); + + code = acosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after ACOS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 1.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before ACOS:%f\n", *((float *)pInput->data)); + + code = acosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after ACOS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, acosFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {3.14159265358979312, 1.57079632679489656, 0.0}; + + //TINYINT + int8_t val_tinyint[] = {-1, 0, 1}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before ACOS:%d\n", *((int8_t *)pInput->data + i)); + } + + code = acosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after ACOS:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-1.0, 0.0, 1.0}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before ACOS:%f\n", *((float *)pInput->data + i)); + } + + code = acosFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after ACOS:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, atanFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 0.78539816339744828; + + //TINYINT + int8_t val_tinyint = 1; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before ATAN:%d\n", *((int8_t *)pInput->data)); + + code = atanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after ATAN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 1.00; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before ATAN:%f\n", *((float *)pInput->data)); + + code = atanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after ATAN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, atanFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {-0.78539816339744828, 0.0, 0.78539816339744828}; + + //TINYINT + int8_t val_tinyint[] = {-1, 0, 1}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before ATAN:%d\n", *((int8_t *)pInput->data + i)); + } + + code = atanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after ATAN:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-1.0, 0.0, 1.0}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before ATAN:%f\n", *((float *)pInput->data + i)); + } + + code = atanFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after ATAN:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, ceilFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result = 10.0; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("tiny_int before CEIL:%d\n", *((int8_t *)pInput->data)); + + code = ceilFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), (int8_t)result); + PRINTF("tiny_int after CEIL:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 9.5; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before CEIL:%f\n", *((float *)pInput->data)); + + code = ceilFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), (float)result); + PRINTF("float after CEIL:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, ceilFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result[] = {-10.0, 0.0, 10.0}; + + //TINYINT + int8_t val_tinyint[] = {-10, 0, 10}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before CEIL:%d\n", *((int8_t *)pInput->data + i)); + } + + code = ceilFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), result[i]); + PRINTF("tiny_int after CEIL:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-10.5, 0.0, 9.5}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before CEIL:%f\n", *((float *)pInput->data + i)); + } + + code = ceilFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), result[i]); + PRINTF("float after CEIL:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, floorFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result = 10.0; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("tiny_int before FLOOR:%d\n", *((int8_t *)pInput->data)); + + code = floorFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), (int8_t)result); + PRINTF("tiny_int after FLOOR:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 10.5; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before FLOOR:%f\n", *((float *)pInput->data)); + + code = floorFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), (float)result); + PRINTF("float after FLOOR:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, floorFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result[] = {-10.0, 0.0, 10.0}; + + //TINYINT + int8_t val_tinyint[] = {-10, 0, 10}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before FLOOR:%d\n", *((int8_t *)pInput->data + i)); + } + + code = floorFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), result[i]); + PRINTF("tiny_int after FLOOR:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-9.5, 0.0, 10.5}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before FLOOR:%f\n", *((float *)pInput->data + i)); + } + + code = floorFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), result[i]); + PRINTF("float after FLOOR:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, roundFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result = 10.0; + + //TINYINT + int8_t val_tinyint = 10; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("tiny_int before ROUND:%d\n", *((int8_t *)pInput->data)); + + code = roundFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), (int8_t)result); + PRINTF("tiny_int after ROUND:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 9.5; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + PRINTF("float before ROUND:%f\n", *((float *)pInput->data)); + + code = roundFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), (float)result); + PRINTF("float after ROUND:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, roundFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + double result[] = {-10.0, 0.0, 10.0}; + + //TINYINT + int8_t val_tinyint[] = {-10, 0, 10}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before ROUND:%d\n", *((int8_t *)pInput->data + i)); + } + + code = roundFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)pOutput->data + i), result[i]); + PRINTF("tiny_int after ROUND:%d\n", *((int8_t *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {-9.5, 0.0, 9.5}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, type, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before ROUND:%f\n", *((float *)pInput->data + i)); + } + + code = roundFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((float *)pOutput->data + i), result[i]); + PRINTF("float after ROUND:%f\n", *((float *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, sqrtFunction_constant) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 5.0; + + //TINYINT + int8_t val_tinyint = 25; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, &val_tinyint, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before SQRT:%d\n", *((int8_t *)pInput->data)); + + code = sqrtFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after SQRT:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float = 25.0; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, &val_float, 1, true); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before SQRT:%f\n", *((float *)pInput->data)); + + code = sqrtFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after SQRT:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, sqrtFunction_column) { + SScalarParam *pInput, *pOutput; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {5.0, 9.0, 10.0}; + + //TINYINT + int8_t val_tinyint[] = {25, 81, 100}; + type = TSDB_DATA_TYPE_TINYINT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput->data + i) = val_tinyint[i]; + PRINTF("tiny_int before SQRT:%d\n", *((int8_t *)pInput->data + i)); + } + + code = sqrtFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after SQRT:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {25.0, 81.0, 100.0}; + type = TSDB_DATA_TYPE_FLOAT; + scltMakeDataBlock(&pInput, type, 0, rowNum, false); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput->data + i) = val_float[i]; + PRINTF("float before SQRT:%f\n", *((float *)pInput->data + i)); + } + + code = sqrtFunction(pInput, 1, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after SQRT:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(pInput); + scltDestroyDataBlock(pOutput); +} + +TEST(ScalarFunctionTest, logFunction_constant) { + SScalarParam *pInput, *pOutput; + SScalarParam *input[2]; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 3.0; + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + + //TINYINT + int8_t val_tinyint[] = {27, 3}; + type = TSDB_DATA_TYPE_TINYINT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, &val_tinyint[i], 1, true); + pInput[i] = *input[i]; + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before LOG: %d,%d\n", *((int8_t *)pInput[0].data), + *((int8_t *)pInput[1].data)); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after LOG:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {64.0, 4.0}; + type = TSDB_DATA_TYPE_FLOAT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, &val_float[i], 1, true); + pInput[i] = *input[i]; + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before LOG: %f,%f\n", *((float *)pInput[0].data), + *((float *)pInput[1].data)); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after LOG:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //TINYINT AND FLOAT + int8_t param0 = 64; + float param1 = 4.0; + scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, ¶m0, 1, true); + pInput[0] = *input[0]; + scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, ¶m1, 1, true); + pInput[1] = *input[1]; + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + PRINTF("tiny_int,float before LOG: %d,%f\n", *((int8_t *)pInput[0].data), *((float *)pInput[1].data)); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int,float after LOG:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + taosMemoryFree(pInput); +} + +TEST(ScalarFunctionTest, logFunction_column) { + SScalarParam *pInput, *pOutput; + SScalarParam *input[2]; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {2.0, 4.0, 3.0}; + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + + //TINYINT + int8_t val_tinyint[2][3] = {{25, 81, 64}, {5, 3, 4}}; + type = TSDB_DATA_TYPE_TINYINT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, 0, rowNum, false); + pInput[i] = *input[i]; + for (int32_t j = 0; j < rowNum; ++j) { + *((int8_t *)pInput[i].data + j) = val_tinyint[i][j]; + } + PRINTF("tiny_int before LOG:%d,%d,%d\n", *((int8_t *)pInput[i].data + 0), + *((int8_t *)pInput[i].data + 1), + *((int8_t *)pInput[i].data + 2)); + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after LOG:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[2][3] = {{25.0, 81.0, 64.0}, {5.0, 3.0, 4.0}}; + type = TSDB_DATA_TYPE_FLOAT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, 0, rowNum, false); + pInput[i] = *input[i]; + for (int32_t j = 0; j < rowNum; ++j) { + *((float *)pInput[i].data + j) = val_float[i][j]; + } + PRINTF("float before LOG:%f,%f,%f\n", *((float *)pInput[i].data + 0), + *((float *)pInput[i].data + 1), + *((float *)pInput[i].data + 2)); + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after LOG:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //TINYINT AND FLOAT + int8_t param0[] = {25, 81, 64}; + float param1[] = {5.0, 3.0, 4.0}; + scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, 0, rowNum, false); + pInput[0] = *input[0]; + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput[0].data + i) = param0[i]; + } + scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, 0, rowNum, false); + pInput[1] = *input[1]; + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput[1].data + i) = param1[i]; + } + PRINTF("tiny_int, float before LOG:{%d,%f}, {%d,%f}, {%d,%f}\n", *((int8_t *)pInput[0].data + 0), *((float *)pInput[1].data + 0), + *((int8_t *)pInput[0].data + 1), *((float *)pInput[1].data + 1), + *((int8_t *)pInput[0].data + 2), *((float *)pInput[1].data + 2)); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = logFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int,float after LOG:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + taosMemoryFree(pInput); +} + +TEST(ScalarFunctionTest, powFunction_constant) { + SScalarParam *pInput, *pOutput; + SScalarParam *input[2]; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result = 16.0; + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + + //TINYINT + int8_t val_tinyint[] = {2, 4}; + type = TSDB_DATA_TYPE_TINYINT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, &val_tinyint[i], 1, true); + pInput[i] = *input[i]; + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("tiny_int before POW: %d,%d\n", *((int8_t *)pInput[0].data), + *((int8_t *)pInput[1].data)); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int after POW:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[] = {2.0, 4.0}; + type = TSDB_DATA_TYPE_FLOAT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, &val_float[i], 1, true); + pInput[i] = *input[i]; + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + PRINTF("float before POW: %f,%f\n", *((float *)pInput[0].data), + *((float *)pInput[1].data)); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("float after POW:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //TINYINT AND FLOAT + int8_t param0 = 2; + float param1 = 4.0; + scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, ¶m0, 1, true); + pInput[0] = *input[0]; + scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, ¶m1, 1, true); + pInput[1] = *input[1]; + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + PRINTF("tiny_int,float before POW: %d,%f\n", *((int8_t *)pInput[0].data), *((float *)pInput[1].data)); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result); + PRINTF("tiny_int,float after POW:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + taosMemoryFree(pInput); +} + +TEST(ScalarFunctionTest, powFunction_column) { + SScalarParam *pInput, *pOutput; + SScalarParam *input[2]; + int32_t code = TSDB_CODE_SUCCESS; + int32_t rowNum = 3; + int32_t type; + int32_t otype = TSDB_DATA_TYPE_DOUBLE; + double result[] = {32.0, 27.0, 16.0}; + pInput = (SScalarParam *)taosMemoryCalloc(2, sizeof(SScalarParam)); + + //TINYINT + int8_t val_tinyint[2][3] = {{2, 3, 4}, {5, 3, 2}}; + type = TSDB_DATA_TYPE_TINYINT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, 0, rowNum, false); + pInput[i] = *input[i]; + for (int32_t j = 0; j < rowNum; ++j) { + *((int8_t *)pInput[i].data + j) = val_tinyint[i][j]; + } + PRINTF("tiny_int before POW:%d,%d,%d\n", *((int8_t *)pInput[i].data + 0), + *((int8_t *)pInput[i].data + 1), + *((int8_t *)pInput[i].data + 2)); + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int after POW:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //FLOAT + float val_float[2][3] = {{2.0, 3.0, 4.0}, {5.0, 3.0, 2.0}}; + type = TSDB_DATA_TYPE_FLOAT; + for (int32_t i = 0; i < 2; ++i) { + scltMakeDataBlock(&input[i], type, 0, rowNum, false); + pInput[i] = *input[i]; + for (int32_t j = 0; j < rowNum; ++j) { + *((float *)pInput[i].data + j) = val_float[i][j]; + } + PRINTF("float before POW:%f,%f,%f\n", *((float *)pInput[i].data + 0), + *((float *)pInput[i].data + 1), + *((float *)pInput[i].data + 2)); + } + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("float after POW:%f\n", *((double *)pOutput->data + i)); + } + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + + //TINYINT AND FLOAT + int8_t param0[] = {2, 3, 4}; + float param1[] = {5.0, 3.0, 2.0}; + scltMakeDataBlock(&input[0], TSDB_DATA_TYPE_TINYINT, 0, rowNum, false); + pInput[0] = *input[0]; + for (int32_t i = 0; i < rowNum; ++i) { + *((int8_t *)pInput[0].data + i) = param0[i]; + } + scltMakeDataBlock(&input[1], TSDB_DATA_TYPE_FLOAT, 0, rowNum, false); + pInput[1] = *input[1]; + for (int32_t i = 0; i < rowNum; ++i) { + *((float *)pInput[1].data + i) = param1[i]; + } + PRINTF("tiny_int, float before POW:{%d,%f}, {%d,%f}, {%d,%f}\n", *((int8_t *)pInput[0].data + 0), *((float *)pInput[1].data + 0), + *((int8_t *)pInput[0].data + 1), *((float *)pInput[1].data + 1), + *((int8_t *)pInput[0].data + 2), *((float *)pInput[1].data + 2)); + scltMakeDataBlock(&pOutput, otype, 0, rowNum, false); + + code = powFunction(pInput, 2, pOutput); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)pOutput->data + i), result[i]); + PRINTF("tiny_int,float after POW:%f\n", *((double *)pOutput->data + i)); + } + + scltDestroyDataBlock(input[0]); + scltDestroyDataBlock(input[1]); + scltDestroyDataBlock(pOutput); + taosMemoryFree(pInput); +} int main(int argc, char** argv) { taosSeedRand(taosGetTimestampSec()); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 1c40f255cf..518da6e2b8 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -25,6 +25,7 @@ extern "C" { #include "planner.h" #include "scheduler.h" #include "thash.h" +#include "trpc.h" #define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 #define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 @@ -44,7 +45,7 @@ typedef struct SSchTrans { typedef struct SSchHbTrans { SRWLatch lock; - uint64_t seqId; + SRpcCtx rpcCtx; SSchTrans trans; } SSchHbTrans; @@ -76,12 +77,23 @@ typedef struct SSchedulerMgmt { SHashObj *hbConnections; } SSchedulerMgmt; -typedef struct SSchCallbackParam { - uint64_t queryId; - int64_t refId; - uint64_t taskId; - void *transport; -} SSchCallbackParam; +typedef struct SSchCallbackParamHeader { + bool isHbParam; +} SSchCallbackParamHeader; + +typedef struct SSchTaskCallbackParam { + SSchCallbackParamHeader head; + uint64_t queryId; + int64_t refId; + uint64_t taskId; + void *transport; +} SSchTaskCallbackParam; + +typedef struct SSchHbCallbackParam { + SSchCallbackParamHeader head; + SQueryNodeEpId nodeEpId; + void *transport; +} SSchHbCallbackParam; typedef struct SSchFlowControl { SRWLatch lock; @@ -91,6 +103,11 @@ typedef struct SSchFlowControl { SArray *taskList; // Element is SSchTask* } SSchFlowControl; +typedef struct SSchNodeInfo { + SQueryNodeAddr addr; + void *handle; +} SSchNodeInfo; + typedef struct SSchLevel { int32_t level; int8_t status; @@ -116,7 +133,7 @@ typedef struct SSchTask { SQueryNodeAddr succeedAddr; // task executed success node address int8_t candidateIdx; // current try condidation index SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr - SArray *execAddrs; // all tried node for current task, element is SQueryNodeAddr + SArray *execNodes; // all tried node for current task, element is SSchNodeInfo SQueryProfileSummary summary; // task execution summary int32_t childReady; // child task ready number SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* @@ -178,6 +195,8 @@ extern SSchedulerMgmt schMgmt; #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) #define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task)) +#define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL) +#define SCH_SET_TASK_HANDLE(_task, _handle) ((_task)->handle = (_handle)) #define SCH_SET_JOB_STATUS(job, st) atomic_store_8(&(job)->status, st) #define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status) @@ -205,6 +224,8 @@ extern SSchedulerMgmt schMgmt; qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__) #define SCH_TASK_DLOG(param, ...) \ qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__) +#define SCH_TASK_DLOGL(param, ...) \ + qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__) #define SCH_TASK_WLOG(param, ...) \ qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__) @@ -228,6 +249,8 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask); int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); int32_t schFetchFromRemote(SSchJob *pJob); int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); +int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId); +int32_t schCloneSMsgSendInfo(void *src, void **dst); #ifdef __cplusplus diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index fe102ed6ed..5af13d97ca 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -18,6 +18,7 @@ #include "schedulerInt.h" #include "tmsg.h" #include "tref.h" +#include "trpc.h" SSchedulerMgmt schMgmt = {0}; @@ -56,21 +57,41 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * pTask->level = pLevel; SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); pTask->taskId = schGenTaskId(); - pTask->execAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); - if (NULL == pTask->execAddrs) { - SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CANDIDATE_EP_NUM); + pTask->execNodes = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SSchNodeInfo)); + if (NULL == pTask->execNodes) { + SCH_TASK_ELOG("taosArrayInit %d execNodes failed", SCH_MAX_CANDIDATE_EP_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; } -void schFreeTask(SSchTask *pTask) { +void schFreeRpcCtx(SRpcCtx *pCtx) { + if (NULL == pCtx) { + return; + } + void *pIter = taosHashIterate(pCtx->args, NULL); + while (pIter) { + SRpcCtxVal *ctxVal = (SRpcCtxVal *)pIter; + + (*ctxVal->freeFunc)(ctxVal->val); + + pIter = taosHashIterate(pCtx->args, pIter); + } + + taosHashCleanup(pCtx->args); + + if (pCtx->brokenVal.freeFunc) { + (*pCtx->brokenVal.freeFunc)(pCtx->brokenVal.val); + } +} + +void schFreeTask(SSchTask* pTask) { if (pTask->candidateAddrs) { taosArrayDestroy(pTask->candidateAddrs); } - tfree(pTask->msg); + taosMemoryFreeClear(pTask->msg); if (pTask->children) { taosArrayDestroy(pTask->children); @@ -80,8 +101,8 @@ void schFreeTask(SSchTask *pTask) { taosArrayDestroy(pTask->parents); } - if (pTask->execAddrs) { - taosArrayDestroy(pTask->execAddrs); + if (pTask->execNodes) { + taosArrayDestroy(pTask->execNodes); } } @@ -99,31 +120,66 @@ static FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { int32_t lastMsgType = SCH_GET_TASK_LASTMSG_TYPE(pTask); int32_t taskStatus = SCH_GET_TASK_STATUS(pTask); - + int32_t reqMsgType = msgType - 1; switch (msgType) { - case TDMT_VND_CREATE_TABLE_RSP: - case TDMT_VND_SUBMIT_RSP: - case TDMT_VND_QUERY_RSP: + case TDMT_SCH_LINK_BROKEN: + return TSDB_CODE_SUCCESS; + case TDMT_VND_QUERY_RSP: // query_rsp may be processed later than ready_rsp + if (lastMsgType != reqMsgType && -1 != lastMsgType && TDMT_VND_FETCH != lastMsgType) { + SCH_TASK_DLOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + } + + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + } + + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + return TSDB_CODE_SUCCESS; case TDMT_VND_RES_READY_RSP: - case TDMT_VND_FETCH_RSP: - case TDMT_VND_DROP_TASK: - if (lastMsgType != (msgType - 1)) { - SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), - TMSG_INFO(msgType)); + reqMsgType = TDMT_VND_QUERY; + if (lastMsgType != reqMsgType && -1 != lastMsgType) { + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", (lastMsgType > 0 ? TMSG_INFO(lastMsgType) : "null"), TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + return TSDB_CODE_SUCCESS; + case TDMT_VND_FETCH_RSP: + if (lastMsgType != reqMsgType && -1 != lastMsgType) { + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + return TSDB_CODE_SUCCESS; + case TDMT_VND_CREATE_TABLE_RSP: + case TDMT_VND_SUBMIT_RSP: break; default: SCH_TASK_ELOG("unknown rsp msg, type:%s, status:%s", TMSG_INFO(msgType), jobTaskStatusStr(taskStatus)); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + if (lastMsgType != reqMsgType) { + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); return TSDB_CODE_SUCCESS; @@ -299,12 +355,16 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr) { - if (NULL == taosArrayPush(pTask->execAddrs, addr)) { - SCH_TASK_ELOG("taosArrayPush addr to execAddr list failed, errno:%d", errno); +int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, void *handle) { + SSchNodeInfo nodeInfo = {.addr = *addr, .handle = handle}; + + if (NULL == taosArrayPush(pTask->execNodes, &nodeInfo)) { + SCH_TASK_ELOG("taosArrayPush nodeInfo to execNodes list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + SCH_TASK_DLOG("task execNode recorded, handle:%p", handle); + return TSDB_CODE_SUCCESS; } @@ -632,53 +692,23 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchHbTrans *trans) { +int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { int32_t code = 0; SSchHbTrans *hb = NULL; - while (true) { - hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); - if (NULL == hb) { - code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), trans, sizeof(SSchHbTrans)); - if (code) { - if (HASH_NODE_EXIST(code)) { - continue; - } - - qError("taosHashPut hb trans failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); - SCH_ERR_RET(code); - } - - qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 - ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", - trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, - trans->trans.transHandle); - - return TSDB_CODE_SUCCESS; - } - - break; + hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } SCH_LOCK(SCH_WRITE, &hb->lock); - - if (hb->seqId >= trans->seqId) { - qDebug("hb trans seqId is old, seqId:%" PRId64 ", currentId:%" PRId64 ", nodeId:%d, fqdn:%s, port:%d", trans->seqId, - hb->seqId, epId->nodeId, epId->ep.fqdn, epId->ep.port); - - SCH_UNLOCK(SCH_WRITE, &hb->lock); - return TSDB_CODE_SUCCESS; - } - - hb->seqId = trans->seqId; - memcpy(&hb->trans, &trans->trans, sizeof(trans->trans)); - + memcpy(&hb->trans, trans, sizeof(*trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); - qDebug("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 - ", nodeId:%d, fqdn:%s, port:%d, instance:%p, connection:%p", - trans->seqId, schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->trans.transInst, - trans->trans.transHandle); + qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, handle:%p", + schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->transInst, + trans->transHandle); return TSDB_CODE_SUCCESS; } @@ -993,7 +1023,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); + //SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); break; } @@ -1019,7 +1049,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch if (pJob->resData) { SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->resData); - tfree(rsp); + taosMemoryFreeClear(rsp); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -1041,6 +1071,10 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); break; } + case TDMT_SCH_LINK_BROKEN: + SCH_TASK_ELOG("link broken received, error:%x - %s", rspCode, tstrerror(rspCode)); + SCH_ERR_JRET(rspCode); + break; default: SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%s", msgType, SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1055,12 +1089,12 @@ _return: int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { int32_t code = 0; - SSchCallbackParam *pParam = (SSchCallbackParam *)param; + SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; SSchTask *pTask = NULL; SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { - qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, pParam->queryId, pParam->taskId, pParam->refId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } @@ -1078,9 +1112,9 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in } pTask = *task; - SCH_TASK_DLOG("rsp msg received, type:%s, code:%s", TMSG_INFO(msgType), tstrerror(rspCode)); + SCH_TASK_DLOG("rsp msg received, type:%s, handle:%p, code:%s", TMSG_INFO(msgType), pMsg->handle, tstrerror(rspCode)); - pTask->handle = pMsg->handle; + SCH_SET_TASK_HANDLE(pTask, pMsg->handle); SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); _return: @@ -1089,7 +1123,7 @@ _return: schReleaseJob(pParam->refId); } - tfree(param); + taosMemoryFreeClear(param); SCH_RET(code); } @@ -1114,7 +1148,7 @@ int32_t schHandleReadyCallback(void *param, const SDataBuf *pMsg, int32_t code) } int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { - SSchCallbackParam *pParam = (SSchCallbackParam *)param; + SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; qDebug("QID:%" PRIx64 ",TID:%" PRIx64 " drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); } @@ -1125,24 +1159,22 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { } SSchedulerHbRsp rsp = {0}; - - SSchCallbackParam *pParam = (SSchCallbackParam *)param; - if (tDeserializeSSchedulerHbRsp(pMsg->pData, pMsg->len, &rsp)) { qError("invalid hb rsp msg, size:%d", pMsg->len); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - if (rsp.seqId != (uint64_t)-1) { - SSchHbTrans trans = {0}; - trans.seqId = rsp.seqId; - trans.trans.transInst = pParam->transport; - trans.trans.transHandle = pMsg->handle; + SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - SCH_RET(schUpdateHbConnection(&rsp.epId, &trans)); - } + SSchTrans trans = {0}; + trans.transInst = pParam->transport; + trans.transHandle = pMsg->handle; + + SCH_ERR_RET(schUpdateHbConnection(&rsp.epId, &trans)); int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); + qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, rsp.epId.ep.port); + for (int32_t i = 0; i < taskNum; ++i) { STaskStatus *taskStatus = taosArrayGet(rsp.taskStatus, i); @@ -1155,6 +1187,8 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { } // TODO + + SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId, jobTaskStatusStr(taskStatus->status)); schReleaseJob(taskStatus->refId); } @@ -1166,6 +1200,26 @@ _return: SCH_RET(code); } +int32_t schHandleLinkBrokenCallback(void *param, const SDataBuf *pMsg, int32_t code) { + SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param; + rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); + + qDebug("handle %p is broken", pMsg->handle); + + if (head->isHbParam) { + SSchHbCallbackParam *hbParam = (SSchHbCallbackParam *)param; + SSchTrans trans = {.transInst = hbParam->transport, .transHandle = NULL}; + SCH_ERR_RET(schUpdateHbConnection(&hbParam->nodeEpId, &trans)); + + SCH_ERR_RET(schBuildAndSendHbMsg(&hbParam->nodeEpId)); + } else { + SCH_ERR_RET(schHandleCallback(param, pMsg, TDMT_SCH_LINK_BROKEN, code)); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { switch (msgType) { case TDMT_VND_CREATE_TABLE: @@ -1189,6 +1243,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { case TDMT_VND_QUERY_HEARTBEAT: *fp = schHandleHbCallback; break; + case TDMT_SCH_LINK_BROKEN: + *fp = schHandleLinkBrokenCallback; + break; default: qError("unknown msg type for callback, msgType:%d", msgType); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -1197,21 +1254,335 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { return TSDB_CODE_SUCCESS; } -int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet *epSet, int32_t msgType, void *msg, - uint32_t msgSize) { +void schFreeRpcCtxVal(const void *arg) { + if (NULL == arg) { + return; + } + + SMsgSendInfo* pMsgSendInfo = (SMsgSendInfo *)arg; + taosMemoryFreeClear(pMsgSendInfo->param); + taosMemoryFreeClear(pMsgSendInfo); +} + +int32_t schMakeTaskCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { + SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); + if (NULL == param) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param->queryId = pJob->queryId; + param->refId = pJob->refId; + param->taskId = SCH_TASK_ID(pTask); + param->transport = pJob->transport; + + *pParam = param; + + return TSDB_CODE_SUCCESS; +} + +int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { + SSchHbCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); + if (NULL == param) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param->head.isHbParam = true; + + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + + param->nodeEpId.nodeId = addr->nodeId; + memcpy(¶m->nodeEpId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + param->transport = pJob->transport; + + *pParam = param; + + return TSDB_CODE_SUCCESS; +} + + +int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb) { + int32_t code = 0; + SMsgSendInfo* pMsgSendInfo = NULL; + + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (isHb) { + SCH_ERR_JRET(schMakeHbCallbackParam(pJob, pTask, &pMsgSendInfo->param)); + } else { + SCH_ERR_JRET(schMakeTaskCallbackParam(pJob, pTask, &pMsgSendInfo->param)); + } + + int32_t msgType = TDMT_SCH_LINK_BROKEN; + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); + + pMsgSendInfo->fp = fp; + + brokenVal->msgType = msgType; + brokenVal->val = pMsgSendInfo; + brokenVal->clone = schCloneSMsgSendInfo; + brokenVal->freeFunc = schFreeRpcCtxVal; + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(pMsgSendInfo->param); + taosMemoryFreeClear(pMsgSendInfo); + + SCH_RET(code); +} + +int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { + int32_t code = 0; + SSchTaskCallbackParam *param = NULL; + SMsgSendInfo* pMsgSendInfo = NULL; + + pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (NULL == pCtx->args) { + SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); + if (NULL == param) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + int32_t msgType = TDMT_VND_RES_READY_RSP; + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(TDMT_VND_RES_READY, &fp)); + + param->queryId = pJob->queryId; + param->refId = pJob->refId; + param->taskId = SCH_TASK_ID(pTask); + param->transport = pJob->transport; + + pMsgSendInfo->param = param; + pMsgSendInfo->fp = fp; + + SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo, .freeFunc = schFreeRpcCtxVal}; + if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { + SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, false)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosHashCleanup(pCtx->args); + taosMemoryFreeClear(param); + taosMemoryFreeClear(pMsgSendInfo); + + SCH_RET(code); +} + +int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { + int32_t code = 0; + SSchHbCallbackParam *param = NULL; + SMsgSendInfo* pMsgSendInfo = NULL; + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SQueryNodeEpId epId = {0}; + + epId.nodeId = addr->nodeId; + memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); + + pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (NULL == pCtx->args) { + SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); + if (NULL == param) { + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + int32_t msgType = TDMT_VND_QUERY_HEARTBEAT_RSP; + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(TDMT_VND_QUERY_HEARTBEAT, &fp)); + + param->nodeEpId = epId; + param->transport = pJob->transport; + + pMsgSendInfo->param = param; + pMsgSendInfo->fp = fp; + + SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo, .freeFunc = schFreeRpcCtxVal}; + if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { + SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, true)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosHashCleanup(pCtx->args); + taosMemoryFreeClear(param); + taosMemoryFreeClear(pMsgSendInfo); + + SCH_RET(code); +} + + +int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { + int32_t code = 0; + SSchHbTrans hb = {0}; + + hb.trans.transInst = pJob->transport; + + SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &hb.rpcCtx)); + + code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), &hb, sizeof(SSchHbTrans)); + if (code) { + schFreeRpcCtx(&hb.rpcCtx); + + if (HASH_NODE_EXIST(code)) { + *exist = true; + return TSDB_CODE_SUCCESS; + } + + qError("taosHashPut hb trans failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_ERR_RET(code); + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHeader **pDst) { + if (pSrc->isHbParam) { + SSchHbCallbackParam *dst = taosMemoryMalloc(sizeof(SSchHbCallbackParam)); + if (NULL == dst) { + qError("malloc SSchHbCallbackParam failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(dst, pSrc, sizeof(*dst)); + *pDst = (SSchCallbackParamHeader *)dst; + + return TSDB_CODE_SUCCESS; + } + + SSchTaskCallbackParam *dst = taosMemoryMalloc(sizeof(SSchTaskCallbackParam)); + if (NULL == dst) { + qError("malloc SSchTaskCallbackParam failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(dst, pSrc, sizeof(*dst)); + *pDst = (SSchCallbackParamHeader *)dst; + + return TSDB_CODE_SUCCESS; +} + +int32_t schCloneSMsgSendInfo(void *src, void **dst) { + SMsgSendInfo *pSrc = src; + int32_t code = 0; + SMsgSendInfo *pDst = taosMemoryMalloc(sizeof(*pSrc)); + if (NULL == pDst) { + qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(pDst, pSrc, sizeof(*pSrc)); + pDst->param = NULL; + + SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); + + *dst = pDst; + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(pDst); + SCH_RET(code); +} + +int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { + int32_t code = 0; + memcpy(&pDst->brokenVal, &pSrc->brokenVal, sizeof(pSrc->brokenVal)); + pDst->brokenVal.val = NULL; + + SCH_ERR_RET(schCloneSMsgSendInfo(pSrc->brokenVal.val, &pDst->brokenVal.val)); + + pDst->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + if (NULL == pDst->args) { + qError("taosHashInit %d RpcCtx failed", 1); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SRpcCtxVal dst = {0}; + void *pIter = taosHashIterate(pSrc->args, NULL); + while (pIter) { + SRpcCtxVal *pVal = (SRpcCtxVal *)pIter; + int32_t *msgType = taosHashGetKey(pIter, NULL); + + dst = *pVal; + dst.val = NULL; + + SCH_ERR_JRET(schCloneSMsgSendInfo(pVal->val, &dst.val)); + + if (taosHashPut(pDst->args, msgType, sizeof(*msgType), &dst, sizeof(dst))) { + qError("taosHashPut msg %d to rpcCtx failed", *msgType); + (*dst.freeFunc)(dst.val); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pIter = taosHashIterate(pSrc->args, pIter); + } + + return TSDB_CODE_SUCCESS; + +_return: + + schFreeRpcCtx(pDst); + SCH_RET(code); +} + + +int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* epSet, int32_t msgType, void *msg, uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) { int32_t code = 0; SSchTrans *trans = (SSchTrans *)transport; - SMsgSendInfo *pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + SMsgSendInfo *pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam)); + SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); if (NULL == param) { - SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchCallbackParam)); + SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1230,8 +1601,12 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = fp; - int64_t transporterId = 0; - code = asyncSendMsgToServer(trans->transInst, epSet, &transporterId, pMsgSendInfo); + qDebug("start to send %s msg to node[%d,%s,%d], refId:%" PRIx64 "instance:%p, handle:%p", + TMSG_INFO(msgType), ntohl(((SMsgHead *)msg)->vgId), epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, + pJob->refId, trans->transInst, trans->transHandle); + + int64_t transporterId = 0; + code = asyncSendMsgToServerExt(trans->transInst, epSet, &transporterId, pMsgSendInfo, persistHandle, ctx); if (code) { SCH_ERR_JRET(code); } @@ -1241,16 +1616,108 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet _return: - tfree(param); - tfree(pMsgSendInfo); + taosMemoryFreeClear(param); + taosMemoryFreeClear(pMsgSendInfo); + SCH_RET(code); +} + +int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { + SSchedulerHbReq req = {0}; + int32_t code = 0; + SRpcCtx rpcCtx = {0}; + SSchTrans trans = {0}; + int32_t msgType = TDMT_VND_QUERY_HEARTBEAT; + + req.header.vgId = htonl(nodeEpId->nodeId); + req.sId = schMgmt.sId; + memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); + + SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, nodeEpId->ep.port); + SCH_ERR_RET(code); + } + + SCH_LOCK(SCH_WRITE, &hb->lock); + code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); + memcpy(&trans, &hb->trans, sizeof(trans)); + SCH_UNLOCK(SCH_WRITE, &hb->lock); + + SCH_ERR_RET(code); + + int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); + if (msgSize < 0) { + qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + void *msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + qError("calloc hb req %d failed", msgSize); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { + qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SMsgSendInfo *pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("calloc SMsgSendInfo failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); + if (NULL == param) { + qError("calloc SSchTaskCallbackParam failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + __async_send_cb_fn_t fp = NULL; + SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); + + param->transport = trans.transInst; + + pMsgSendInfo->param = param; + pMsgSendInfo->msgInfo.pData = msg; + pMsgSendInfo->msgInfo.len = msgSize; + pMsgSendInfo->msgInfo.handle = trans.transHandle; + pMsgSendInfo->msgType = msgType; + pMsgSendInfo->fp = fp; + + int64_t transporterId = 0; + SEpSet epSet = {.inUse = 0, .numOfEps = 1}; + memcpy(&epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); + + qDebug("start to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d", trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port); + + code = asyncSendMsgToServerExt(trans.transInst, &epSet, &transporterId, pMsgSendInfo, true, &rpcCtx); + if (code) { + qError("fail to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d, error:%x - %s", + trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port, code, tstrerror(code)); + SCH_ERR_JRET(code); + } + + qDebug("hb msg sent"); + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(msg); + taosMemoryFreeClear(param); + taosMemoryFreeClear(pMsgSendInfo); + schFreeRpcCtx(&rpcCtx); SCH_RET(code); } int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void *msg = NULL; - int32_t code = 0; - bool isCandidateAddr = false; + void *msg = NULL; + int32_t code = 0; + bool isCandidateAddr = false; + bool persistHandle = false; + SRpcCtx rpcCtx = {0}; + if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); isCandidateAddr = true; @@ -1262,7 +1729,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_CREATE_TABLE: case TDMT_VND_SUBMIT: { msgSize = pTask->msgLen; - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1273,10 +1740,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_QUERY: { + SCH_ERR_RET(schMakeQueryRpcCtx(pJob, pTask, &rpcCtx)); + uint32_t len = strlen(pJob->sql); - msgSize = sizeof(SSubQueryMsg) + pTask->msgLen + len; - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1294,13 +1762,14 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, memcpy(pMsg->msg, pJob->sql, len); memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); - + + persistHandle = true; break; } case TDMT_VND_RES_READY: { msgSize = sizeof(SResReadyReq); - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1317,7 +1786,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_FETCH: { msgSize = sizeof(SResFetchReq); - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1330,11 +1799,12 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(pJob->queryId); pMsg->taskId = htobe64(pTask->taskId); + break; } case TDMT_VND_DROP_TASK: { msgSize = sizeof(STaskDropReq); - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1351,6 +1821,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, break; } case TDMT_VND_QUERY_HEARTBEAT: { + SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &rpcCtx)); + SSchedulerHbReq req = {0}; req.sId = schMgmt.sId; req.header.vgId = addr->nodeId; @@ -1362,7 +1834,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - msg = calloc(1, msgSize); + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_JOB_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1371,6 +1843,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + + persistHandle = true; break; } default: @@ -1381,11 +1855,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_SET_TASK_LASTMSG_TYPE(pTask, msgType); - SSchTrans trans = {.transInst = pJob->transport, .transHandle = pTask ? pTask->handle : NULL}; - SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize)); + SSchTrans trans = {.transInst = pJob->transport, .transHandle = SCH_GET_TASK_HANDLE(pTask)}; + SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL))); - if (isCandidateAddr) { - SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr)); + if (msgType == TDMT_VND_QUERY) { + SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr, trans.transHandle)); } return TSDB_CODE_SUCCESS; @@ -1393,8 +1867,9 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, _return: SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); + schFreeRpcCtx(&rpcCtx); - tfree(msg); + taosMemoryFreeClear(msg); SCH_RET(code); } @@ -1405,10 +1880,16 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { epId.nodeId = addr->nodeId; memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp)); +#if 1 SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, &epId, sizeof(SQueryNodeEpId)); if (NULL == hb) { - SCH_ERR_RET(schBuildAndSendMsg(pJob, NULL, addr, TDMT_VND_QUERY_HEARTBEAT)); + bool exist = false; + SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId, &exist)); + if (!exist) { + SCH_ERR_RET(schBuildAndSendHbMsg(&epId)); + } } +#endif return TSDB_CODE_SUCCESS; } @@ -1440,7 +1921,7 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { pTask->msgLen); SCH_ERR_RET(code); } else { - SCH_TASK_DLOG("physical plan len:%d, %s", pTask->msgLen, pTask->msg); + SCH_TASK_DLOGL("physical plan len:%d, %s", pTask->msgLen, pTask->msg); } } @@ -1460,6 +1941,8 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { bool enough = false; int32_t code = 0; + SCH_SET_TASK_HANDLE(pTask, NULL); + if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { SCH_ERR_JRET(schCheckIncTaskFlowQuota(pJob, pTask, &enough)); @@ -1500,23 +1983,24 @@ int32_t schLaunchJob(SSchJob *pJob) { } void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { - if (NULL == pTask->execAddrs) { + if (NULL == pTask->execNodes) { SCH_TASK_DLOG("no exec address, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); return; } - int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); + int32_t size = (int32_t)taosArrayGetSize(pTask->execNodes); if (size <= 0) { - SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); + SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); return; } - SQueryNodeAddr *addr = NULL; + SSchNodeInfo *nodeInfo = NULL; for (int32_t i = 0; i < size; ++i) { - addr = (SQueryNodeAddr *)taosArrayGet(pTask->execAddrs, i); + nodeInfo = (SSchNodeInfo *)taosArrayGet(pTask->execNodes, i); + SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); - schBuildAndSendMsg(pJob, pTask, addr, TDMT_VND_DROP_TASK); + schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_VND_DROP_TASK); } SCH_TASK_DLOG("task has %d exec address", size); @@ -1588,8 +2072,8 @@ void schFreeJobImpl(void *job) { taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); - tfree(pJob->resData); - tfree(pJob); + taosMemoryFreeClear(pJob->resData); + taosMemoryFreeClear(pJob); qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); } @@ -1603,7 +2087,7 @@ static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryPlan *pD } int32_t code = 0; - SSchJob *pJob = calloc(1, sizeof(SSchJob)); + SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:%" PRIx64 " calloc %d failed", pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1790,7 +2274,7 @@ int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SSubQueryMsg* pMsg = calloc(1, msgSize); + SSubQueryMsg* pMsg = taosMemoryCalloc(1, msgSize); pMsg->header.vgId = tInfo.addr.nodeId; @@ -1807,7 +2291,7 @@ int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks) { if (NULL == taosArrayPush(info, &tInfo)) { qError("taosArrayPush failed, idx:%d", i); - free(msg); + taosMemoryFree(msg); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } @@ -1839,7 +2323,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { info.addr = src->addr; for (int32_t i = 0; i < copyNum; ++i) { - info.msg = malloc(msgSize); + info.msg = taosMemoryMalloc(msgSize); if (NULL == info.msg) { qError("malloc %d failed", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1851,7 +2335,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { if (NULL == taosArrayPush(*dst, &info)) { qError("taosArrayPush failed, idx:%d", i); - free(info.msg); + taosMemoryFree(info.msg); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } } @@ -1931,7 +2415,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { } if (NULL == *pData) { - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); if (rsp) { rsp->completed = 1; } @@ -1993,7 +2477,7 @@ void schedulerFreeTaskList(SArray *taskList) { int32_t taskNum = taosArrayGetSize(taskList); for (int32_t i = 0; i < taskNum; ++i) { STaskInfo *info = taosArrayGet(taskList, i); - tfree(info->msg); + taosMemoryFreeClear(info->msg); } taosArrayDestroy(taskList); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index b5bd68ba86..cf04b06579 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -95,8 +95,8 @@ void schtBuildQueryDag(SQueryPlan *dag) { SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SSubplan *scanPlan = (SSubplan *)calloc(1, sizeof(SSubplan)); - SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); + SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); scanPlan->id.queryId = qId; scanPlan->id.groupId = 0x0000000000000002; @@ -110,7 +110,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = nodesMakeList(); - scanPlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + scanPlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan->msgType = TDMT_VND_QUERY; mergePlan->id.queryId = qId; @@ -122,7 +122,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { mergePlan->pChildren = nodesMakeList(); mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_VND_QUERY; merge->pNodeList = nodesMakeList(); @@ -148,8 +148,8 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { SNodeListNode *scan = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); SNodeListNode *merge = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SSubplan *scanPlan = (SSubplan *)calloc(scanPlanNum, sizeof(SSubplan)); - SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + SSubplan *scanPlan = (SSubplan *)taosMemoryCalloc(scanPlanNum, sizeof(SSubplan)); + SSubplan *mergePlan = (SSubplan *)taosMemoryCalloc(1, sizeof(SSubplan)); merge->pNodeList = nodesMakeList(); scan->pNodeList = nodesMakeList(); @@ -173,7 +173,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { scanPlan[i].pChildren = NULL; scanPlan[i].level = 1; scanPlan[i].pParents = nodesMakeList(); - scanPlan[i].pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + scanPlan[i].pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); scanPlan[i].msgType = TDMT_VND_QUERY; nodesListAppend(scanPlan[i].pParents, (SNode*)mergePlan); @@ -190,7 +190,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { mergePlan->execNode.epSet.numOfEps = 0; mergePlan->pParents = NULL; - mergePlan->pNode = (SPhysiNode*)calloc(1, sizeof(SPhysiNode)); + mergePlan->pNode = (SPhysiNode*)taosMemoryCalloc(1, sizeof(SPhysiNode)); mergePlan->msgType = TDMT_VND_QUERY; nodesListAppend(merge->pNodeList, (SNode*)mergePlan); @@ -213,7 +213,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { dag->pSubplans = nodesMakeList(); SNodeListNode *inserta = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); - SSubplan *insertPlan = (SSubplan *)calloc(2, sizeof(SSubplan)); + SSubplan *insertPlan = (SSubplan *)taosMemoryCalloc(2, sizeof(SSubplan)); insertPlan[0].id.queryId = qId; insertPlan[0].id.groupId = 0x0000000000000003; @@ -228,7 +228,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[0].pChildren = NULL; insertPlan[0].pParents = NULL; insertPlan[0].pNode = NULL; - insertPlan[0].pDataSink = (SDataSinkNode*)calloc(1, sizeof(SDataSinkNode)); + insertPlan[0].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[0].msgType = TDMT_VND_SUBMIT; insertPlan[1].id.queryId = qId; @@ -244,7 +244,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { insertPlan[1].pChildren = NULL; insertPlan[1].pParents = NULL; insertPlan[1].pNode = NULL; - insertPlan[1].pDataSink = (SDataSinkNode*)calloc(1, sizeof(SDataSinkNode)); + insertPlan[1].pDataSink = (SDataSinkNode*)taosMemoryCalloc(1, sizeof(SDataSinkNode)); insertPlan[1].msgType = TDMT_VND_SUBMIT; inserta->pNodeList = nodesMakeList(); @@ -258,7 +258,7 @@ void schtBuildInsertDag(SQueryPlan *dag) { int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { - *str = (char *)calloc(1, 20); + *str = (char *)taosMemoryCalloc(1, 20); *len = 20; return 0; } @@ -312,9 +312,9 @@ void schtSetRpcSendRequest() { int32_t schtAsyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo) { if (pInfo) { - tfree(pInfo->param); - tfree(pInfo->msgInfo.pData); - free(pInfo); + taosMemoryFreeClear(pInfo->param); + taosMemoryFreeClear(pInfo->msgInfo.pData); + taosMemoryFree(pInfo); } return 0; } @@ -373,7 +373,7 @@ void *schtCreateFetchRspThread(void *param) { taosSsleep(1); int32_t code = 0; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; @@ -387,7 +387,7 @@ void *schtCreateFetchRspThread(void *param) { void *schtFetchRspThread(void *aa) { SDataBuf dataBuf = {0}; - SSchCallbackParam* param = NULL; + SSchTaskCallbackParam* param = NULL; while (!schtTestStop) { if (0 == atomic_val_compare_exchange_32(&schtStartFetch, 1, 0)) { @@ -396,13 +396,13 @@ void *schtFetchRspThread(void *aa) { taosUsleep(1); - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->queryId = schtQueryId; param->taskId = schtFetchTaskId; int32_t code = 0; - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); rsp->completed = 1; rsp->numOfRows = 10; @@ -449,7 +449,7 @@ void* schtRunJobThread(void *aa) { schtSetAsyncSendMsgToServer(); SSchJob *pJob = NULL; - SSchCallbackParam *param = NULL; + SSchTaskCallbackParam *param = NULL; SHashObj *execTasks = NULL; SDataBuf dataBuf = {0}; uint32_t jobFinished = 0; @@ -484,7 +484,7 @@ void* schtRunJobThread(void *aa) { pIter = taosHashIterate(pJob->execTasks, pIter); } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -504,7 +504,7 @@ void* schtRunJobThread(void *aa) { } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -524,7 +524,7 @@ void* schtRunJobThread(void *aa) { } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -544,7 +544,7 @@ void* schtRunJobThread(void *aa) { } - param = (SSchCallbackParam *)calloc(1, sizeof(*param)); + param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param)); param->refId = queryJobRefId; param->queryId = pJob->queryId; @@ -697,7 +697,7 @@ TEST(queryTest, normalCase) { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; ASSERT_EQ(pRsp->completed, 1); ASSERT_EQ(pRsp->numOfRows, 10); - tfree(data); + taosMemoryFreeClear(data); data = NULL; code = schedulerFetchRows(job, &data); @@ -713,6 +713,116 @@ TEST(queryTest, normalCase) { schedulerDestroy(); } +TEST(queryTest, readyFirstCase) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + SVgroupInfo vgInfo = {0}; + int64_t job = 0; + SQueryPlan dag; + + memset(&dag, 0, sizeof(dag)); + + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); + + SEp qnodeAddr = {0}; + strcpy(qnodeAddr.fqdn, "qnode0.ep"); + qnodeAddr.port = 6031; + taosArrayPush(qnodeList, &qnodeAddr); + + int32_t code = schedulerInit(NULL); + ASSERT_EQ(code, 0); + + schtBuildQueryDag(&dag); + + schtSetPlanToString(); + schtSetExecNode(); + schtSetAsyncSendMsgToServer(); + + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job); + ASSERT_EQ(code, 0); + + + SSchJob *pJob = schAcquireJob(job); + + void *pIter = taosHashIterate(pJob->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SResReadyRsp rsp = {0}; + code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); + printf("code:%d", code); + ASSERT_EQ(code, 0); + pIter = taosHashIterate(pJob->execTasks, pIter); + } + + pIter = taosHashIterate(pJob->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SQueryTableRsp rsp = {0}; + code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); + + ASSERT_EQ(code, 0); + pIter = taosHashIterate(pJob->execTasks, pIter); + } + + pIter = taosHashIterate(pJob->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SResReadyRsp rsp = {0}; + code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0); + ASSERT_EQ(code, 0); + + pIter = taosHashIterate(pJob->execTasks, pIter); + } + + pIter = taosHashIterate(pJob->execTasks, NULL); + while (pIter) { + SSchTask *task = *(SSchTask **)pIter; + + SQueryTableRsp rsp = {0}; + code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0); + + ASSERT_EQ(code, 0); + pIter = taosHashIterate(pJob->execTasks, pIter); + } + + + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + + TdThread thread1; + taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + + void *data = NULL; + code = schedulerFetchRows(job, &data); + ASSERT_EQ(code, 0); + + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; + ASSERT_EQ(pRsp->completed, 1); + ASSERT_EQ(pRsp->numOfRows, 10); + taosMemoryFreeClear(data); + + data = NULL; + code = schedulerFetchRows(job, &data); + ASSERT_EQ(code, 0); + ASSERT_TRUE(data == NULL); + + schReleaseJob(job); + + schedulerFreeJob(job); + + schtFreeQueryDag(&dag); + + schedulerDestroy(); +} + + + TEST(queryTest, flowCtrlCase) { void *mockPointer = (void *)0x1; char *clusterId = "cluster1"; @@ -793,7 +903,7 @@ TEST(queryTest, flowCtrlCase) { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; ASSERT_EQ(pRsp->completed, 1); ASSERT_EQ(pRsp->numOfRows, 10); - tfree(data); + taosMemoryFreeClear(data); data = NULL; code = schedulerFetchRows(job, &data); diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt new file mode 100644 index 0000000000..572c70d31b --- /dev/null +++ b/source/libs/stream/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src STREAM_SRC) +add_library(stream STATIC ${STREAM_SRC}) +target_include_directories( + stream + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/stream" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + stream + PRIVATE os util transport qcom executor +) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/libs/stream/inc/tstreamInc.h b/source/libs/stream/inc/tstreamInc.h new file mode 100644 index 0000000000..c96901e567 --- /dev/null +++ b/source/libs/stream/inc/tstreamInc.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _TSTREAM_H_ +#define _TSTREAM_H_ + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef _TSTREAM_H_ */ diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c new file mode 100644 index 0000000000..3ec6603673 --- /dev/null +++ b/source/libs/stream/src/tstream.c @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tstream.h" +#include "executor.h" + +int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) { + SArray* pRes = NULL; + // source + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK && pTask->sourceType != TASK_SOURCE__SCAN) return 0; + + // exec + if (pTask->execType != TASK_EXEC__NONE) { + ASSERT(workId < pTask->exec.numOfRunners); + void* exec = pTask->exec.runners[workId].executor; + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) { + return -1; + } + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + qSetStreamInput(exec, input, inputType); + while (1) { + SSDataBlock* output; + uint64_t ts; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) { + break; + } + taosArrayPush(pRes, output); + } + } else if (inputType == STREAM_DATA_TYPE_SSDATA_BLOCK) { + const SArray* blocks = (const SArray*)input; + int32_t sz = taosArrayGetSize(blocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pBlock = taosArrayGet(blocks, i); + qSetStreamInput(exec, pBlock, inputType); + while (1) { + SSDataBlock* output; + uint64_t ts; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(false); + } + if (output == NULL) { + break; + } + taosArrayPush(pRes, output); + } + } + } else { + ASSERT(0); + } + } else { + ASSERT(inputType == STREAM_DATA_TYPE_SSDATA_BLOCK); + pRes = (SArray*)input; + } + + // sink + if (pTask->sinkType == TASK_SINK__TABLE) { + // + } else if (pTask->sinkType == TASK_SINK__SMA) { + // + } else if (pTask->sinkType == TASK_SINK__FETCH) { + // + } else if (pTask->sinkType == TASK_SINK__SHOW) { + blockDebugShowData(pRes); + } else { + ASSERT(pTask->sinkType == TASK_SINK__NONE); + } + + // dispatch + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + SStreamTaskExecReq req = { + .streamId = pTask->streamId, + .taskId = pTask->taskId, + .data = pRes, + }; + + int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req); + void* buf = rpcMallocCont(tlen); + + if (buf == NULL) { + return -1; + } + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSStreamTaskExecReq(&abuf, &req); + + SRpcMsg dispatchMsg = { + .pCont = buf, + .contLen = tlen, + .code = 0, + .msgType = pTask->dispatchMsgType, + }; + + int32_t qType; + if (pTask->dispatchMsgType == TDMT_VND_TASK_PIPE_EXEC || pTask->dispatchMsgType == TDMT_SND_TASK_PIPE_EXEC) { + qType = FETCH_QUEUE; + } else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC || + pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) { + qType = MERGE_QUEUE; + } else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) { + qType = WRITE_QUEUE; + } else { + ASSERT(0); + } + tmsgPutToQueue(pMsgCb, qType, &dispatchMsg); + + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + SStreamTaskExecReq req = { + .streamId = pTask->streamId, + .taskId = pTask->taskId, + .data = pRes, + }; + + int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req); + void* buf = rpcMallocCont(tlen); + + if (buf == NULL) { + return -1; + } + + ((SMsgHead*)buf)->vgId = htonl(pTask->fixedEpDispatcher.nodeId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSStreamTaskExecReq(&abuf, &req); + + SRpcMsg dispatchMsg = { + .pCont = buf, + .contLen = tlen, + .code = 0, + .msgType = pTask->dispatchMsgType, + }; + + SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet; + + tmsgSendReq(pMsgCb, pEpSet, &dispatchMsg); + + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + // TODO + + } else { + ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); + } + return 0; +} + +int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->streamId); + tlen += taosEncodeFixedI32(buf, pReq->taskId); + tlen += tEncodeDataBlocks(buf, pReq->data); + return tlen; +} + +void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->streamId); + buf = taosDecodeFixedI32(buf, &pReq->taskId); + buf = tDecodeDataBlocks(buf, &pReq->data); + return (void*)buf; +} + +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { taosArrayDestroy(pReq->data); } + +SStreamTask* tNewSStreamTask(int64_t streamId) { + SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); + if (pTask == NULL) { + return NULL; + } + pTask->taskId = tGenIdPI32(); + pTask->streamId = streamId; + pTask->status = STREAM_TASK_STATUS__RUNNING; + /*pTask->qmsg = NULL;*/ + return pTask; +} + +int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask) { + /*if (tStartEncode(pEncoder) < 0) return -1;*/ + if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1; + if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->downstreamTaskId) < 0) return -1; + + if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; + + if (pTask->execType != TASK_EXEC__NONE) { + if (tEncodeI8(pEncoder, pTask->exec.parallelizable) < 0) return -1; + if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1; + } + + if (pTask->sinkType != TASK_SINK__NONE) { + // TODO: wrap + if (tEncodeI8(pEncoder, pTask->tbSink.reserved) < 0) return -1; + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + if (tEncodeI8(pEncoder, pTask->inplaceDispatcher.reserved) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + if (tEncodeI8(pEncoder, pTask->shuffleDispatcher.hashMethod) < 0) return -1; + } + + /*tEndEncode(pEncoder);*/ + return pEncoder->pos; +} + +int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask) { + /*if (tStartDecode(pDecoder) < 0) return -1;*/ + if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1; + if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->downstreamTaskId) < 0) return -1; + + if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; + + if (pTask->execType != TASK_EXEC__NONE) { + if (tDecodeI8(pDecoder, &pTask->exec.parallelizable) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1; + } + + if (pTask->sinkType != TASK_SINK__NONE) { + if (tDecodeI8(pDecoder, &pTask->tbSink.reserved) < 0) return -1; + } + + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { + if (tDecodeI8(pDecoder, &pTask->inplaceDispatcher.reserved) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; + } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + if (tDecodeI8(pDecoder, &pTask->shuffleDispatcher.hashMethod) < 0) return -1; + } + + /*tEndDecode(pDecoder);*/ + return 0; +} + +void tFreeSStreamTask(SStreamTask* pTask) { + // TODO + /*taosMemoryFree(pTask->qmsg);*/ + /*taosMemoryFree(pTask->executor);*/ + /*taosMemoryFree(pTask);*/ +} + +#if 0 +int32_t tEncodeSStreamTaskExecReq(SCoder* pEncoder, const SStreamTaskExecReq* pReq) { + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + /*if (tEncodeDataBlocks(buf, pReq->streamId) < 0) return -1;*/ + return pEncoder->size; +} +int32_t tDecodeSStreamTaskExecReq(SCoder* pDecoder, SStreamTaskExecReq* pReq) { + return pEncoder->size; +} +void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { + taosArrayDestroyEx(pReq->data, tDeleteSSDataBlock); +} +#endif diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index d47a525e25..726ebdae07 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -74,7 +74,7 @@ static void syncEnvTick(void *param, void *tmrId) { } static SSyncEnv *doSyncEnvStart() { - SSyncEnv *pSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + SSyncEnv *pSyncEnv = (SSyncEnv *)taosMemoryMalloc(sizeof(SSyncEnv)); assert(pSyncEnv != NULL); memset(pSyncEnv, 0, sizeof(SSyncEnv)); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index da54aa9668..717b1cc756 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -121,7 +121,7 @@ int32_t syncIOPingTimerStop() { // local function ------------ static SSyncIO *syncIOCreate(char *host, uint16_t port) { - SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + SSyncIO *io = (SSyncIO *)taosMemoryMalloc(sizeof(SSyncIO)); memset(io, 0, sizeof(*io)); io->pMsgQ = taosOpenQueue(); diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index aa97104180..d33075054a 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -19,7 +19,7 @@ // SMatchIndex ----------------------------- SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { - SSyncIndexMgr *pSyncIndexMgr = malloc(sizeof(SSyncIndexMgr)); + SSyncIndexMgr *pSyncIndexMgr = taosMemoryMalloc(sizeof(SSyncIndexMgr)); assert(pSyncIndexMgr != NULL); memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr)); @@ -33,7 +33,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { if (pSyncIndexMgr != NULL) { - free(pSyncIndexMgr); + taosMemoryFree(pSyncIndexMgr); } } @@ -78,12 +78,12 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i])); } int respondNum = 0; - int *arr = (int *)malloc(sizeof(int) * pSyncIndexMgr->replicaNum); + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pSyncIndexMgr->replicaNum); for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { arr[i] = pSyncIndexMgr->index[i]; } cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum); - free(arr); + taosMemoryFree(arr); cJSON_AddItemToObject(pRoot, "index", pIndex); snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); @@ -106,24 +106,24 @@ void syncIndexMgrPrint(SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); printf("syncIndexMgrPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncIndexMgrPrint2(char *s, SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); printf("syncIndexMgrPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncIndexMgrLog(SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); sTrace("syncIndexMgrLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncIndexMgrLog2(char *s, SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 58e02b3866..78e454309a 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -151,7 +151,7 @@ int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seq // open/close -------------- SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { - SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); @@ -310,7 +310,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncNodeStopElectTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode); - free(pSyncNode); + taosMemoryFree(pSyncNode); } // ping -------------- @@ -723,26 +723,26 @@ void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); printf("syncNodePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncNodePrint2(char* s, SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); printf("syncNodePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncNodeLog(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); sTrace("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncNodeLog2(char* s, SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); sTrace("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ------ local funciton --------- @@ -828,7 +828,7 @@ static int32_t syncNodeEqNoop(SSyncNode* ths) { syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); ths->FpEqMsg(ths->queue, &rpcMsg); - free(serialized); + taosMemoryFree(serialized); syncClientRequestDestroy(pSyncMsg); return ret; @@ -928,5 +928,5 @@ static void syncFreeNode(void* param) { SSyncNode* pNode = param; syncNodePrint2((char*)"==syncFreeNode==", pNode); - free(pNode); + taosMemoryFree(pNode); } \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index f2344c6b6d..5dca165455 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -70,20 +70,20 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont2", s); - free(s); + taosMemoryFree(s); } else { pRoot = cJSON_CreateObject(); char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); cJSON_AddStringToObject(pRoot, "pCont2", s); - free(s); + taosMemoryFree(s); } cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); @@ -118,32 +118,32 @@ void syncRpcMsgPrint(SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); printf("syncRpcMsgPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); printf("syncRpcMsgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRpcMsgLog(SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); sTrace("syncRpcMsgLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncTimeout---- SyncTimeout* syncTimeoutBuild() { uint32_t bytes = sizeof(SyncTimeout); - SyncTimeout* pMsg = malloc(bytes); + SyncTimeout* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_TIMEOUT; @@ -161,7 +161,7 @@ SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock void syncTimeoutDestroy(SyncTimeout* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -176,7 +176,7 @@ void syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg) { } char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncTimeoutSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -187,7 +187,7 @@ char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len) { SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncTimeout* pMsg = malloc(bytes); + SyncTimeout* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncTimeoutDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -243,32 +243,32 @@ void syncTimeoutPrint(const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); printf("syncTimeoutPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); printf("syncTimeoutPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncTimeoutLog(const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); sTrace("syncTimeoutLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncTimeoutLog2(char* s, const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncPing---- SyncPing* syncPingBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncPing) + dataLen; - SyncPing* pMsg = malloc(bytes); + SyncPing* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_PING; @@ -292,7 +292,7 @@ SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId) { void syncPingDestroy(SyncPing* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -308,7 +308,7 @@ void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) { } char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncPingSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -319,7 +319,7 @@ char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len) { SyncPing* syncPingDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncPing* pMsg = malloc(bytes); + SyncPing* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncPingDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -385,10 +385,10 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -408,32 +408,32 @@ void syncPingPrint(const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); printf("syncPingPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingPrint2(char* s, const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); printf("syncPingPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingLog(const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); sTrace("syncPingLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncPingLog2(char* s, const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncPingReply---- SyncPingReply* syncPingReplyBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncPingReply) + dataLen; - SyncPingReply* pMsg = malloc(bytes); + SyncPingReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_PING_REPLY; @@ -457,7 +457,7 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId) void syncPingReplyDestroy(SyncPingReply* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -473,7 +473,7 @@ void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg } char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncPingReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -484,7 +484,7 @@ char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len) { SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncPingReply* pMsg = malloc(bytes); + SyncPingReply* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncPingReplyDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -550,10 +550,10 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -573,32 +573,32 @@ void syncPingReplyPrint(const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); printf("syncPingReplyPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); printf("syncPingReplyPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncPingReplyLog(const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); sTrace("syncPingReplyLog | len:%zu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncPingReplyLog2(char* s, const SyncPingReply* pMsg) { char* serialized = syncPingReply2Str(pMsg); sTrace("syncPingReplyLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncClientRequest---- SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncClientRequest) + dataLen; - SyncClientRequest* pMsg = malloc(bytes); + SyncClientRequest* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_CLIENT_REQUEST; @@ -620,7 +620,7 @@ SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint6 void syncClientRequestDestroy(SyncClientRequest* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -635,7 +635,7 @@ void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientReque } char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncClientRequestSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -646,7 +646,7 @@ char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncClientRequest* pMsg = malloc(bytes); + SyncClientRequest* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncClientRequestDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -688,10 +688,10 @@ cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -711,32 +711,32 @@ void syncClientRequestPrint(const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); printf("syncClientRequestPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); printf("syncClientRequestPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncClientRequestLog(const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); sTrace("syncClientRequestLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncRequestVote---- SyncRequestVote* syncRequestVoteBuild() { uint32_t bytes = sizeof(SyncRequestVote); - SyncRequestVote* pMsg = malloc(bytes); + SyncRequestVote* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_REQUEST_VOTE; @@ -745,7 +745,7 @@ SyncRequestVote* syncRequestVoteBuild() { void syncRequestVoteDestroy(SyncRequestVote* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -760,7 +760,7 @@ void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* } char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncRequestVoteSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -771,7 +771,7 @@ char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len) { SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncRequestVote* pMsg = malloc(bytes); + SyncRequestVote* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncRequestVoteDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -857,32 +857,32 @@ void syncRequestVotePrint(const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); printf("syncRequestVotePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); printf("syncRequestVotePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteLog(const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); sTrace("syncRequestVoteLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncRequestVoteReply---- SyncRequestVoteReply* syncRequestVoteReplyBuild() { uint32_t bytes = sizeof(SyncRequestVoteReply); - SyncRequestVoteReply* pMsg = malloc(bytes); + SyncRequestVoteReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_REQUEST_VOTE_REPLY; @@ -891,7 +891,7 @@ SyncRequestVoteReply* syncRequestVoteReplyBuild() { void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -906,7 +906,7 @@ void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestV } char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncRequestVoteReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -917,7 +917,7 @@ char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncRequestVoteReply* pMsg = malloc(bytes); + SyncRequestVoteReply* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncRequestVoteReplyDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -1000,32 +1000,32 @@ void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); printf("syncRequestVoteReplyPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); printf("syncRequestVoteReplyPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); sTrace("syncRequestVoteReplyLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncAppendEntries---- SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncAppendEntries) + dataLen; - SyncAppendEntries* pMsg = malloc(bytes); + SyncAppendEntries* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_APPEND_ENTRIES; @@ -1035,7 +1035,7 @@ SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { void syncAppendEntriesDestroy(SyncAppendEntries* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -1051,7 +1051,7 @@ void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntri } char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncAppendEntriesSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -1062,7 +1062,7 @@ char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len) SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncAppendEntries* pMsg = malloc(bytes); + SyncAppendEntries* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncAppendEntriesDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -1140,10 +1140,10 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { char* s; s = syncUtilprintBin((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pMsg->data), pMsg->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -1163,32 +1163,32 @@ void syncAppendEntriesPrint(const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); printf("syncAppendEntriesPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); printf("syncAppendEntriesPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesLog(const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); sTrace("syncAppendEntriesLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // ---- message process SyncAppendEntriesReply---- SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { uint32_t bytes = sizeof(SyncAppendEntriesReply); - SyncAppendEntriesReply* pMsg = malloc(bytes); + SyncAppendEntriesReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY; @@ -1197,7 +1197,7 @@ SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg) { if (pMsg != NULL) { - free(pMsg); + taosMemoryFree(pMsg); } } @@ -1212,7 +1212,7 @@ void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppend } char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len) { - char* buf = malloc(pMsg->bytes); + char* buf = taosMemoryMalloc(pMsg->bytes); assert(buf != NULL); syncAppendEntriesReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { @@ -1223,7 +1223,7 @@ char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint3 SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SyncAppendEntriesReply* pMsg = malloc(bytes); + SyncAppendEntriesReply* pMsg = taosMemoryMalloc(bytes); assert(pMsg != NULL); syncAppendEntriesReplyDeserialize(buf, len, pMsg); assert(len == pMsg->bytes); @@ -1309,24 +1309,24 @@ void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); printf("syncAppendEntriesReplyPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); printf("syncAppendEntriesReplyPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); sTrace("syncAppendEntriesReplyLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 9b9842b81d..3de39a6aa8 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -18,7 +18,7 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SSyncRaftEntry) + dataLen; - SSyncRaftEntry* pEntry = malloc(bytes); + SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); assert(pEntry != NULL); memset(pEntry, 0, bytes); pEntry->bytes = bytes; @@ -63,13 +63,13 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index) { void syncEntryDestory(SSyncRaftEntry* pEntry) { if (pEntry != NULL) { - free(pEntry); + taosMemoryFree(pEntry); } } // step 5. SSyncRaftEntry => bin, to raft log char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { - char* buf = malloc(pEntry->bytes); + char* buf = taosMemoryMalloc(pEntry->bytes); assert(buf != NULL); memcpy(buf, pEntry, pEntry->bytes); if (len != NULL) { @@ -81,7 +81,7 @@ char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { // step 6. bin => SSyncRaftEntry, from raft log SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); - SSyncRaftEntry* pEntry = malloc(bytes); + SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); assert(pEntry != NULL); memcpy(pEntry, buf, len); assert(len == pEntry->bytes); @@ -109,11 +109,11 @@ cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { char* s; s = syncUtilprintBin((char*)(pEntry->data), pEntry->dataLen); cJSON_AddStringToObject(pRoot, "data", s); - free(s); + taosMemoryFree(s); s = syncUtilprintBin2((char*)(pEntry->data), pEntry->dataLen); cJSON_AddStringToObject(pRoot, "data2", s); - free(s); + taosMemoryFree(s); } cJSON* pJson = cJSON_CreateObject(); @@ -142,24 +142,24 @@ void syncEntryPrint(const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void syncEntryLog(const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { char* serialized = syncEntry2Str(pObj); sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index b28f899aa9..63c6265d10 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -17,10 +17,10 @@ #include "wal.h" SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { - SSyncLogStore* pLogStore = malloc(sizeof(SSyncLogStore)); + SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore)); assert(pLogStore != NULL); - pLogStore->data = malloc(sizeof(SSyncLogStoreData)); + pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData)); assert(pLogStore->data != NULL); SSyncLogStoreData* pData = pLogStore->data; @@ -39,8 +39,8 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { void logStoreDestory(SSyncLogStore* pLogStore) { if (pLogStore != NULL) { - free(pLogStore->data); - free(pLogStore); + taosMemoryFree(pLogStore->data); + taosMemoryFree(pLogStore); } } @@ -62,7 +62,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { assert(walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len) == 0); walFsync(pWal, true); - free(serialized); + taosMemoryFree(serialized); return code; } @@ -103,7 +103,7 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { SSyncRaftEntry* pLastEntry = logStoreGetLastEntry(pLogStore); if (pLastEntry != NULL) { lastTerm = pLastEntry->term; - free(pLastEntry); + taosMemoryFree(pLastEntry); } return lastTerm; } @@ -202,26 +202,26 @@ void logStorePrint(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); printf("logStorePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void logStorePrint2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); printf("logStorePrint | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void logStoreLog(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); sTrace("logStorePrint | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void logStoreLog2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); sTrace("logStorePrint | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // for debug ----------------- @@ -229,24 +229,24 @@ void logStoreSimplePrint(SSyncLogStore* pLogStore) { char* serialized = logStoreSimple2Str(pLogStore); printf("logStoreSimplePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStoreSimple2Str(pLogStore); printf("logStoreSimplePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void logStoreSimpleLog(SSyncLogStore* pLogStore) { char* serialized = logStoreSimple2Str(pLogStore); sTrace("logStoreSimpleLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStoreSimple2Str(pLogStore); sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 3f6db129ce..c9054d088e 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -26,7 +26,7 @@ static bool raftStoreFileExist(char *path); SRaftStore *raftStoreOpen(const char *path) { int32_t ret; - SRaftStore *pRaftStore = malloc(sizeof(SRaftStore)); + SRaftStore *pRaftStore = taosMemoryMalloc(sizeof(SRaftStore)); if (pRaftStore == NULL) { sError("raftStoreOpen malloc error"); return NULL; @@ -75,7 +75,7 @@ int32_t raftStoreClose(SRaftStore *pRaftStore) { assert(pRaftStore != NULL); taosCloseFile(&pRaftStore->pFile); - free(pRaftStore); + taosMemoryFree(pRaftStore); pRaftStore = NULL; return 0; } @@ -128,7 +128,7 @@ int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { assert(len2 < len); memset(buf, 0, len); snprintf(buf, len, "%s", serialized); - free(serialized); + taosMemoryFree(serialized); cJSON_Delete(pRoot); return 0; @@ -226,23 +226,23 @@ void raftStorePrint(SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); printf("raftStorePrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void raftStorePrint2(char *s, SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); printf("raftStorePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void raftStoreLog(SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); sTrace("raftStoreLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void raftStoreLog2(char *s, SRaftStore *pObj) { char *serialized = raftStore2Str(pObj); sTrace("raftStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 943b268cd3..3ced7a18bc 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -86,7 +86,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { assert(len == pEntry->bytes); memcpy(pMsg->data, serialized, len); - free(serialized); + taosMemoryFree(serialized); syncEntryDestory(pEntry); } else { diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 8fc17ccb51..287500a587 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -76,10 +76,10 @@ bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId = // ---- SSyncBuffer ----- void syncUtilbufBuild(SSyncBuffer* syncBuf, size_t len) { syncBuf->len = len; - syncBuf->data = malloc(syncBuf->len); + syncBuf->data = taosMemoryMalloc(syncBuf->len); } -void syncUtilbufDestroy(SSyncBuffer* syncBuf) { free(syncBuf->data); } +void syncUtilbufDestroy(SSyncBuffer* syncBuf) { taosMemoryFree(syncBuf->data); } void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest) { dest->len = src->len; @@ -88,7 +88,7 @@ void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest) { void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { dest->len = src->len; - dest->data = malloc(dest->len); + dest->data = taosMemoryMalloc(dest->len); memcpy(dest->data, src->data, dest->len); } @@ -161,7 +161,7 @@ bool syncUtilCanPrint(char c) { } char* syncUtilprintBin(char* ptr, uint32_t len) { - char* s = malloc(len + 1); + char* s = taosMemoryMalloc(len + 1); assert(s != NULL); memset(s, 0, len + 1); memcpy(s, ptr, len); @@ -176,7 +176,7 @@ char* syncUtilprintBin(char* ptr, uint32_t len) { char* syncUtilprintBin2(char* ptr, uint32_t len) { uint32_t len2 = len * 4 + 1; - char* s = malloc(len2); + char* s = taosMemoryMalloc(len2); assert(s != NULL); memset(s, 0, len2); diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 69a4c48f39..733dfd05b6 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -23,7 +23,7 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { } SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { - SVotesGranted *pVotesGranted = malloc(sizeof(SVotesGranted)); + SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); assert(pVotesGranted != NULL); memset(pVotesGranted, 0, sizeof(SVotesGranted)); @@ -41,7 +41,7 @@ SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { void voteGrantedDestroy(SVotesGranted *pVotesGranted) { if (pVotesGranted != NULL) { - free(pVotesGranted); + taosMemoryFree(pVotesGranted); } } @@ -89,12 +89,12 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { for (int i = 0; i < pVotesGranted->replicaNum; ++i) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); } - int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum); + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pVotesGranted->replicaNum); for (int i = 0; i < pVotesGranted->replicaNum; ++i) { arr[i] = pVotesGranted->isGranted[i]; } cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); - free(arr); + taosMemoryFree(arr); cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); @@ -126,31 +126,31 @@ void voteGrantedPrint(SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); printf("voteGrantedPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void voteGrantedPrint2(char *s, SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); printf("voteGrantedPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void voteGrantedLog(SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); sTrace("voteGrantedLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void voteGrantedLog2(char *s, SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); sTrace("voteGrantedLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } // SVotesRespond ----------------------------- SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { - SVotesRespond *pVotesRespond = malloc(sizeof(SVotesRespond)); + SVotesRespond *pVotesRespond = taosMemoryMalloc(sizeof(SVotesRespond)); assert(pVotesRespond != NULL); memset(pVotesRespond, 0, sizeof(SVotesRespond)); @@ -164,7 +164,7 @@ SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { void votesRespondDestory(SVotesRespond *pVotesRespond) { if (pVotesRespond != NULL) { - free(pVotesRespond); + taosMemoryFree(pVotesRespond); } } @@ -213,7 +213,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); } int respondNum = 0; - int *arr = (int *)malloc(sizeof(int) * pVotesRespond->replicaNum); + int *arr = (int *)taosMemoryMalloc(sizeof(int) * pVotesRespond->replicaNum); for (int i = 0; i < pVotesRespond->replicaNum; ++i) { arr[i] = pVotesRespond->isRespond[i]; if (pVotesRespond->isRespond[i]) { @@ -221,7 +221,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { } } cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); - free(arr); + taosMemoryFree(arr); cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); @@ -248,24 +248,24 @@ void votesRespondPrint(SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); printf("votesRespondPrint | len:%lu | %s \n", strlen(serialized), serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void votesRespondPrint2(char *s, SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); printf("votesRespondPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); - free(serialized); + taosMemoryFree(serialized); } void votesRespondLog(SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); sTrace("votesRespondLog | len:%lu | %s", strlen(serialized), serialized); - free(serialized); + taosMemoryFree(serialized); } void votesRespondLog2(char *s, SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); sTrace("votesRespondLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); - free(serialized); + taosMemoryFree(serialized); } diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index 362da67c66..72aeb155e7 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -34,14 +34,14 @@ void test1() { void test2() { SyncAppendEntriesReply *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncAppendEntriesReplySerialize(pMsg, serialized, len); SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(); syncAppendEntriesReplyDeserialize(serialized, len, pMsg2); syncAppendEntriesReplyPrint2((char *)"test2: syncAppendEntriesReplySerialize -> syncAppendEntriesReplyDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesReplyDestroy(pMsg); syncAppendEntriesReplyDestroy(pMsg2); } @@ -54,7 +54,7 @@ void test3() { syncAppendEntriesReplyPrint2((char *)"test3: syncAppendEntriesReplySerialize3 -> syncAppendEntriesReplyDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesReplyDestroy(pMsg); syncAppendEntriesReplyDestroy(pMsg2); } diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index 687d9bcb94..69a0aee9a8 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -36,13 +36,13 @@ void test1() { void test2() { SyncAppendEntries *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncAppendEntriesSerialize(pMsg, serialized, len); SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen); syncAppendEntriesDeserialize(serialized, len, pMsg2); syncAppendEntriesPrint2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesDestroy(pMsg); syncAppendEntriesDestroy(pMsg2); } @@ -54,7 +54,7 @@ void test3() { SyncAppendEntries *pMsg2 = syncAppendEntriesDeserialize2(serialized, len); syncAppendEntriesPrint2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncAppendEntriesDestroy(pMsg); syncAppendEntriesDestroy(pMsg2); } @@ -63,7 +63,7 @@ void test4() { SyncAppendEntries *pMsg = createMsg(); SRpcMsg rpcMsg; syncAppendEntries2RpcMsg(pMsg, &rpcMsg); - SyncAppendEntries *pMsg2 = (SyncAppendEntries *)malloc(rpcMsg.contLen); + SyncAppendEntries *pMsg2 = (SyncAppendEntries *)taosMemoryMalloc(rpcMsg.contLen); syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2); syncAppendEntriesPrint2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncClientRequestTest.cpp b/source/libs/sync/test/syncClientRequestTest.cpp index 6323f53a03..f22478d538 100644 --- a/source/libs/sync/test/syncClientRequestTest.cpp +++ b/source/libs/sync/test/syncClientRequestTest.cpp @@ -34,13 +34,13 @@ void test1() { void test2() { SyncClientRequest *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncClientRequestSerialize(pMsg, serialized, len); SyncClientRequest *pMsg2 = syncClientRequestBuild(pMsg->dataLen); syncClientRequestDeserialize(serialized, len, pMsg2); syncClientRequestPrint2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncClientRequestDestroy(pMsg); syncClientRequestDestroy(pMsg2); } @@ -52,7 +52,7 @@ void test3() { SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); syncClientRequestPrint2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncClientRequestDestroy(pMsg); syncClientRequestDestroy(pMsg2); } @@ -61,7 +61,7 @@ void test4() { SyncClientRequest *pMsg = createMsg(); SRpcMsg rpcMsg; syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = (SyncClientRequest *)malloc(rpcMsg.contLen); + SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); syncClientRequestPrint2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 99cfe75867..251a4b538f 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -88,7 +88,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncElectTest2.cpp b/source/libs/sync/test/syncElectTest2.cpp index 8463ffd448..709d9d8580 100644 --- a/source/libs/sync/test/syncElectTest2.cpp +++ b/source/libs/sync/test/syncElectTest2.cpp @@ -91,7 +91,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncElectTest3.cpp b/source/libs/sync/test/syncElectTest3.cpp index 45c5488c60..e36a2d0fb1 100644 --- a/source/libs/sync/test/syncElectTest3.cpp +++ b/source/libs/sync/test/syncElectTest3.cpp @@ -94,7 +94,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index e554858072..b1ba95dc3e 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -87,16 +87,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0() { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 32; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); return pMsg; } @@ -107,7 +107,7 @@ SyncClientRequest *step1(const SRpcMsg *pMsg) { } SRpcMsg *step2(const SyncClientRequest *pMsg) { - SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pRetMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); syncClientRequest2RpcMsg(pMsg, pRetMsg); return pRetMsg; } @@ -133,7 +133,7 @@ SSyncRaftEntry *step6(const char *pMsg, uint32_t len) { } SRpcMsg *step7(const SSyncRaftEntry *pMsg) { - SRpcMsg *pRetMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pRetMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); syncEntry2OriginalRpc(pMsg, pRetMsg); return pRetMsg; } @@ -190,7 +190,7 @@ int main(int argc, char **argv) { char * pMsg5 = step5(pMsg4, &len); char * s = syncUtilprintBin(pMsg5, len); printf("==step5== [%s] \n", s); - free(s); + taosMemoryFree(s); // step6 SSyncRaftEntry *pMsg6 = step6(pMsg5, len); diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index 57315f40ec..48cc7caf2f 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -66,7 +66,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index 2c8433426a..e8427e8168 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -78,7 +78,7 @@ void test4() { SSyncRaftEntry* pEntry2 = syncEntryDeserialize(serialized, len); syncEntryPrint(pEntry2); - free(serialized); + taosMemoryFree(serialized); syncEntryDestory(pEntry2); syncEntryDestory(pEntry); } diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index 0fc3ebfe4c..913e57e59a 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -66,7 +66,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 6bad8f09cf..319ea3e15a 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -68,7 +68,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -93,7 +93,7 @@ int main(int argc, char** argv) { char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); initRaftId(pSyncNode); @@ -105,7 +105,7 @@ int main(int argc, char** argv) { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100); @@ -117,7 +117,7 @@ int main(int argc, char** argv) { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } printf("---------------------------------------\n"); @@ -132,7 +132,7 @@ int main(int argc, char** argv) { char* serialized = syncIndexMgr2Str(pSyncIndexMgr); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } syncIndexMgrDestroy(pSyncIndexMgr); diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index a3e5f41c85..48b5488e41 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index c1cb66f382..a5adba9a88 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -116,7 +116,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncPingReplyTest.cpp b/source/libs/sync/test/syncPingReplyTest.cpp index 8e1448e781..40592eab6f 100644 --- a/source/libs/sync/test/syncPingReplyTest.cpp +++ b/source/libs/sync/test/syncPingReplyTest.cpp @@ -33,13 +33,13 @@ void test1() { void test2() { SyncPingReply *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncPingReplySerialize(pMsg, serialized, len); SyncPingReply *pMsg2 = syncPingReplyBuild(pMsg->dataLen); syncPingReplyDeserialize(serialized, len, pMsg2); syncPingReplyPrint2((char *)"test2: syncPingReplySerialize -> syncPingReplyDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } @@ -51,7 +51,7 @@ void test3() { SyncPingReply *pMsg2 = syncPingReplyDeserialize2(serialized, len); syncPingReplyPrint2((char *)"test3: syncPingReplySerialize3 -> syncPingReplyDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingReplyDestroy(pMsg); syncPingReplyDestroy(pMsg2); } @@ -60,7 +60,7 @@ void test4() { SyncPingReply *pMsg = createMsg(); SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); - SyncPingReply *pMsg2 = (SyncPingReply *)malloc(rpcMsg.contLen); + SyncPingReply *pMsg2 = (SyncPingReply *)taosMemoryMalloc(rpcMsg.contLen); syncPingReplyFromRpcMsg(&rpcMsg, pMsg2); syncPingReplyPrint2((char *)"test4: syncPingReply2RpcMsg -> syncPingReplyFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp index 05e4d99cb0..677518c6ac 100644 --- a/source/libs/sync/test/syncPingSelfTest.cpp +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 83394b0e77..eb774d77c3 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -33,13 +33,13 @@ void test1() { void test2() { SyncPing *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncPingSerialize(pMsg, serialized, len); SyncPing *pMsg2 = syncPingBuild(pMsg->dataLen); syncPingDeserialize(serialized, len, pMsg2); syncPingPrint2((char *)"test2: syncPingSerialize -> syncPingDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingDestroy(pMsg); syncPingDestroy(pMsg2); } @@ -51,7 +51,7 @@ void test3() { SyncPing *pMsg2 = syncPingDeserialize2(serialized, len); syncPingPrint2((char *)"test3: syncPingSerialize3 -> syncPingDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncPingDestroy(pMsg); syncPingDestroy(pMsg2); } @@ -60,7 +60,7 @@ void test4() { SyncPing *pMsg = createMsg(); SRpcMsg rpcMsg; syncPing2RpcMsg(pMsg, &rpcMsg); - SyncPing *pMsg2 = (SyncPing *)malloc(rpcMsg.contLen); + SyncPing *pMsg2 = (SyncPing *)taosMemoryMalloc(rpcMsg.contLen); syncPingFromRpcMsg(&rpcMsg, pMsg2); syncPingPrint2((char *)"test4: syncPing2RpcMsg -> syncPingFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index 20d4a9ce58..3c5e76ca14 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp index 2a041f3f5d..554f67d365 100644 --- a/source/libs/sync/test/syncPingTimerTest2.cpp +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -65,7 +65,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 688802625a..460ff90f4f 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -25,7 +25,7 @@ void initRaftId() { ids[i].vgId = 1234; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } diff --git a/source/libs/sync/test/syncRefTest.cpp b/source/libs/sync/test/syncRefTest.cpp index 8e9adbbc15..96062b1a91 100644 --- a/source/libs/sync/test/syncRefTest.cpp +++ b/source/libs/sync/test/syncRefTest.cpp @@ -33,7 +33,7 @@ typedef struct SyncObj { static void syncFreeObj(void *param) { SyncObj *pObj = (SyncObj *)param; printf("syncFreeObj name:%s rid:%ld \n", pObj->name, pObj->rid); - free(pObj); + taosMemoryFree(pObj); } int32_t init() { @@ -54,7 +54,7 @@ void cleanup() { } int64_t start() { - SyncObj *pObj = (SyncObj *)malloc(sizeof(SyncObj)); + SyncObj *pObj = (SyncObj *)taosMemoryMalloc(sizeof(SyncObj)); assert(pObj != NULL); pObj->data = &g; diff --git a/source/libs/sync/test/syncReplicateLoadTest.cpp b/source/libs/sync/test/syncReplicateLoadTest.cpp index b2d3f1a98d..0841083b4a 100644 --- a/source/libs/sync/test/syncReplicateLoadTest.cpp +++ b/source/libs/sync/test/syncReplicateLoadTest.cpp @@ -53,7 +53,7 @@ void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, boo } void initFsm() { - pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -126,16 +126,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0(int i) { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 128; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); return pMsg; } diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 09aec6534f..9783bac7e5 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -54,7 +54,7 @@ void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, boo } void initFsm() { - pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -121,16 +121,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0(int i) { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 128; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); return pMsg; } diff --git a/source/libs/sync/test/syncReplicateTest2.cpp b/source/libs/sync/test/syncReplicateTest2.cpp index 8739a8d12f..430df7eebd 100644 --- a/source/libs/sync/test/syncReplicateTest2.cpp +++ b/source/libs/sync/test/syncReplicateTest2.cpp @@ -53,7 +53,7 @@ void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, boo } void initFsm() { - pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -126,16 +126,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0(int i) { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 128; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); return pMsg; } @@ -181,7 +181,7 @@ int main(int argc, char **argv) { syncPropose(rid, pMsg0, true); taosMsleep(1000); - free(pMsg0); + taosMemoryFree(pMsg0); sTrace( "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " diff --git a/source/libs/sync/test/syncReplicateTest3.cpp b/source/libs/sync/test/syncReplicateTest3.cpp index ad2172ccd2..792b9c94cf 100644 --- a/source/libs/sync/test/syncReplicateTest3.cpp +++ b/source/libs/sync/test/syncReplicateTest3.cpp @@ -55,7 +55,7 @@ void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, boo } void initFsm() { - pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -128,16 +128,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0(int i) { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 128; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "value-%u-%d", ports[myIndex], i); return pMsg; } @@ -193,7 +193,7 @@ int main(int argc, char **argv) { syncPropose(rid, pMsg0, true); taosMsleep(sleepMS); - free(pMsg0); + taosMemoryFree(pMsg0); sTrace( "syncPropose sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, " diff --git a/source/libs/sync/test/syncRequestVoteReplyTest.cpp b/source/libs/sync/test/syncRequestVoteReplyTest.cpp index 2bce3e4cd6..9e0ebee313 100644 --- a/source/libs/sync/test/syncRequestVoteReplyTest.cpp +++ b/source/libs/sync/test/syncRequestVoteReplyTest.cpp @@ -34,13 +34,13 @@ void test1() { void test2() { SyncRequestVoteReply *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncRequestVoteReplySerialize(pMsg, serialized, len); SyncRequestVoteReply *pMsg2 = syncRequestVoteReplyBuild(); syncRequestVoteReplyDeserialize(serialized, len, pMsg2); syncRequestVoteReplyPrint2((char *)"test2: syncRequestVoteReplySerialize -> syncRequestVoteReplyDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteReplyDestroy(pMsg); syncRequestVoteReplyDestroy(pMsg2); } @@ -53,7 +53,7 @@ void test3() { syncRequestVoteReplyPrint2((char *)"test3: syncRequestVoteReplySerialize3 -> syncRequestVoteReplyDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteReplyDestroy(pMsg); syncRequestVoteReplyDestroy(pMsg2); } diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 22f47046de..6b2120e124 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -35,13 +35,13 @@ void test1() { void test2() { SyncRequestVote *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncRequestVoteSerialize(pMsg, serialized, len); SyncRequestVote *pMsg2 = syncRequestVoteBuild(); syncRequestVoteDeserialize(serialized, len, pMsg2); syncRequestVotePrint2((char *)"test2: syncRequestVoteSerialize -> syncRequestVoteDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteDestroy(pMsg); syncRequestVoteDestroy(pMsg2); } @@ -53,7 +53,7 @@ void test3() { SyncRequestVote *pMsg2 = syncRequestVoteDeserialize2(serialized, len); syncRequestVotePrint2((char *)"test3: syncRequestVoteSerialize3 -> syncRequestVoteDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncRequestVoteDestroy(pMsg); syncRequestVoteDestroy(pMsg2); } diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index 3f46ab5c7c..ab36f42d0d 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -30,13 +30,13 @@ void test1() { void test2() { SyncTimeout *pMsg = createMsg(); uint32_t len = pMsg->bytes; - char * serialized = (char *)malloc(len); + char * serialized = (char *)taosMemoryMalloc(len); syncTimeoutSerialize(pMsg, serialized, len); SyncTimeout *pMsg2 = syncTimeoutBuild(); syncTimeoutDeserialize(serialized, len, pMsg2); syncTimeoutPrint2((char *)"test2: syncTimeoutSerialize -> syncTimeoutDeserialize ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncTimeoutDestroy(pMsg); syncTimeoutDestroy(pMsg2); } @@ -48,7 +48,7 @@ void test3() { SyncTimeout *pMsg2 = syncTimeoutDeserialize2(serialized, len); syncTimeoutPrint2((char *)"test3: syncTimeoutSerialize3 -> syncTimeoutDeserialize2 ", pMsg2); - free(serialized); + taosMemoryFree(serialized); syncTimeoutDestroy(pMsg); syncTimeoutDestroy(pMsg2); } @@ -57,7 +57,7 @@ void test4() { SyncTimeout *pMsg = createMsg(); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pMsg, &rpcMsg); - SyncTimeout *pMsg2 = (SyncTimeout *)malloc(rpcMsg.contLen); + SyncTimeout *pMsg2 = (SyncTimeout *)taosMemoryMalloc(rpcMsg.contLen); syncTimeoutFromRpcMsg(&rpcMsg, pMsg2); syncTimeoutPrint2((char *)"test4: syncTimeout2RpcMsg -> syncTimeoutFromRpcMsg ", pMsg2); diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index 588eb32ffd..6164aed351 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -67,7 +67,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -92,7 +92,7 @@ int main(int argc, char** argv) { char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); initRaftId(pSyncNode); @@ -104,7 +104,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } SyncTerm term = 1234; @@ -114,7 +114,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } for (int i = 0; i < replicaNum; ++i) { @@ -129,7 +129,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } voteGrantedVote(pVotesGranted, reply); @@ -137,7 +137,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } } @@ -147,7 +147,7 @@ int main(int argc, char** argv) { char* serialized = voteGranted2Str(pVotesGranted); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } voteGrantedDestroy(pVotesGranted); diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index 76fd6fab4e..39ba6253a1 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -67,7 +67,7 @@ void initRaftId(SSyncNode* pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char* s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } @@ -92,7 +92,7 @@ int main(int argc, char** argv) { char* serialized = syncNode2Str(pSyncNode); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); initRaftId(pSyncNode); @@ -104,7 +104,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } SyncTerm term = 1234; @@ -114,7 +114,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } for (int i = 0; i < replicaNum; ++i) { @@ -129,7 +129,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } votesRespondAdd(pVotesRespond, reply); @@ -137,7 +137,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } } @@ -147,7 +147,7 @@ int main(int argc, char** argv) { char* serialized = votesRespond2Str(pVotesRespond); assert(serialized != NULL); printf("%s\n", serialized); - free(serialized); + taosMemoryFree(serialized); } votesRespondDestory(pVotesRespond); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 732bcf140b..a3c24f69ff 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -54,7 +54,7 @@ void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SyncIndex index, boo } void initFsm() { - pFsm = (SSyncFSM *)malloc(sizeof(SSyncFSM)); + pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -118,16 +118,16 @@ void initRaftId(SSyncNode *pSyncNode) { ids[i] = pSyncNode->replicasId[i]; char *s = syncUtilRaftId2Str(&ids[i]); printf("raftId[%d] : %s\n", i, s); - free(s); + taosMemoryFree(s); } } SRpcMsg *step0() { - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg)); + SRpcMsg *pMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); pMsg->msgType = 9999; pMsg->contLen = 32; - pMsg->pCont = malloc(pMsg->contLen); + pMsg->pCont = taosMemoryMalloc(pMsg->contLen); snprintf((char *)(pMsg->pCont), pMsg->contLen, "hello, world"); return pMsg; } diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 978649499a..a9b56d42b8 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -8,7 +8,6 @@ target_sources(tdb "src/db/tdbBtree.c" "src/db/tdbDb.c" "src/db/tdbEnv.c" - # "src/db/tdbPage.c" "src/page/tdbPage.c" "src/page/tdbPageL.c" ) diff --git a/source/client/stream/stream.c b/source/libs/tdb/src/btree/tdbBtreeBalance.c similarity index 100% rename from source/client/stream/stream.c rename to source/libs/tdb/src/btree/tdbBtreeBalance.c diff --git a/source/libs/tdb/src/btree/tdbBtreeCommon.c b/source/libs/tdb/src/btree/tdbBtreeCommon.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeCommon.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeDelete.c b/source/libs/tdb/src/btree/tdbBtreeDelete.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeDelete.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeInsert.c b/source/libs/tdb/src/btree/tdbBtreeInsert.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeInsert.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeInt.h b/source/libs/tdb/src/btree/tdbBtreeInt.h new file mode 100644 index 0000000000..b8a935a614 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeInt.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_BTREE_INT_H_ +#define _TDB_BTREE_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_BTREE_INT_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeOpen.c b/source/libs/tdb/src/btree/tdbBtreeOpen.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeOpen.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6e8c9dca9..5980c2b531 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,14 +39,24 @@ struct SBTree { u8 *pTmp; }; +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; + +#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pData[0] +#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pData[0] = (flags)) + typedef struct __attribute__((__packed__)) { - SPgno rChild; -} SBtPageHdr; + TDB_BTREE_PAGE_COMMON_HDR +} SLeafHdr; + +typedef struct __attribute__((__packed__)) { + TDB_BTREE_PAGE_COMMON_HDR; + SPgno pgno; // right-most child +} SIntHdr; typedef struct { - u16 flags; + u8 flags; SBTree *pBt; -} SBtreeZeroPageArg; +} SBtreeInitPageArg; typedef struct { int kLen; @@ -57,7 +67,7 @@ typedef struct { u8 *pTmpSpace; } SCellDecoder; -static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst); +static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst); static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbBtreeOpenImpl(SBTree *pBt); static int tdbBtreeZeroPage(SPage *pPage, void *arg); @@ -65,7 +75,11 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg); static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); -static int tdbBtreeBalance(SBtCursor *pCur); +static int tdbBtreeBalance(SBTC *pCur); +static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); +static int tdbBtcMoveToNext(SBTC *pBtc); +static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno); +static int tdbBtcMoveUpward(SBTC *pBtc); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -120,16 +134,7 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt) { - pCur->pBt = pBt; - pCur->iPage = -1; - pCur->pPage = NULL; - pCur->idx = -1; - - return 0; -} - -int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { +int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { int ret; int idx; SPager *pPager; @@ -145,7 +150,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } if (pCur->idx == -1) { - ASSERT(TDB_PAGE_NCELLS(pCur->pPage) == 0); + ASSERT(TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0); idx = 0; } else { if (cret > 0) { @@ -176,7 +181,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } // Insert the cell to the index - ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell); + ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell, 0); if (ret < 0) { return -1; } @@ -192,12 +197,36 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p return 0; } -static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { - // TODO +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { + SBTC btc; + SCell *pCell; + int cret; + void *pVal; + SCellDecoder cd; + + tdbBtcOpen(&btc, pBt); + + tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); + + if (cret) { + return cret; + } + + pCell = tdbPageGetCell(btc.pPage, btc.idx); + tdbBtreeDecodeCell(btc.pPage, pCell, &cd); + + *vLen = cd.vLen; + pVal = TDB_REALLOC(*ppVal, *vLen); + if (pVal == NULL) { + return -1; + } + + *ppVal = pVal; + memcpy(*ppVal, cd.pVal, cd.vLen); return 0; } -static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst) { +static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) { int ret; SBTree *pBt; SPager *pPager; @@ -218,9 +247,9 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p pCur->iPage = 0; - if (TDB_PAGE_NCELLS(pCur->pPage) == 0) { + if (TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0) { // Current page is empty - ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); + // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); return 0; } @@ -231,7 +260,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p SCellDecoder cd = {0}; pPage = pCur->pPage; - nCells = TDB_PAGE_NCELLS(pPage); + nCells = TDB_PAGE_TOTAL_CELLS(pPage); lidx = 0; ridx = nCells - 1; @@ -242,7 +271,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p midx = (lidx + ridx) >> 1; - pCell = TDB_PAGE_CELL_AT(pPage, midx); + pCell = tdbPageGetCell(pPage, midx); ret = tdbBtreeDecodeCell(pPage, pCell, &cd); if (ret < 0) { // TODO: handle error @@ -265,8 +294,8 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } // Move downward or break - u16 flags = TDB_PAGE_FLAGS(pPage); - u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + u8 flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (leaf) { pCur->idx = midx; *pCRst = c; @@ -274,18 +303,16 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } else { if (c <= 0) { pCur->idx = midx; - tdbBtCursorMoveToChild(pCur, cd.pgno); + tdbBtcMoveDownward(pCur, cd.pgno); } else { + pCur->idx = midx + 1; if (midx == nCells - 1) { /* Move to right-most child */ - pCur->idx = midx + 1; - tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); + tdbBtcMoveDownward(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); } else { - // TODO: reset cd as uninitialized - pCur->idx = midx + 1; - pCell = TDB_PAGE_CELL_AT(pPage, midx + 1); + pCell = tdbPageGetCell(pPage, pCur->idx); tdbBtreeDecodeCell(pPage, pCell, &cd); - tdbBtCursorMoveToChild(pCur, cd.pgno); + tdbBtcMoveDownward(pCur, cd.pgno); } } } @@ -299,32 +326,6 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p return 0; } -static int tdbBtCursorMoveToRoot(SBtCursor *pCur) { - SBTree *pBt; - SPager *pPager; - SPage *pPage; - int ret; - - pBt = pCur->pBt; - pPager = pBt->pPager; - - // pPage = tdbPagerGet(pPager, pBt->root, true); - // if (pPage == NULL) { - // // TODO: handle error - // } - - // ret = tdbInitBtPage(pPage, &pBtPage); - // if (ret < 0) { - // // TODO - // return 0; - // } - - // pCur->pPage = pBtPage; - // pCur->iPage = 0; - - return 0; -} - static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { int mlen; int cret; @@ -363,7 +364,7 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { } // Try to create a new database - SBtreeZeroPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; + SBtreeInitPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg); if (ret < 0) { return -1; @@ -379,28 +380,19 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; - u16 flags; + u8 flags; u8 isLeaf; pBt = (SBTree *)arg; - - flags = TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); - if (isLeaf) { - pPage->szAmHdr = 0; - } else { - pPage->szAmHdr = sizeof(SBtPageHdr); - } - pPage->pPageHdr = pPage->pData; - pPage->pAmHdr = pPage->pPageHdr + pPage->pPageMethods->szPageHdr; - pPage->pCellIdx = pPage->pAmHdr + pPage->szAmHdr; - pPage->pFreeStart = pPage->pCellIdx + pPage->pPageMethods->szOffset * TDB_PAGE_NCELLS(pPage); - pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); - pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + + ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); + + tdbPageInit(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); TDB_BTREE_ASSERT_FLAG(flags); - // Init other fields if (isLeaf) { pPage->kLen = pBt->keyLen; pPage->vLen = pBt->valLen; @@ -413,30 +405,38 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { pPage->minLocal = pBt->minLocal; } - // TODO: need to update the SPage.nFree - pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; - pPage->nOverflow = 0; - return 0; } static int tdbBtreeZeroPage(SPage *pPage, void *arg) { - u16 flags; + u8 flags; SBTree *pBt; + u8 isLeaf; - flags = ((SBtreeZeroPageArg *)arg)->flags; - pBt = ((SBtreeZeroPageArg *)arg)->pBt; + flags = ((SBtreeInitPageArg *)arg)->flags; + pBt = ((SBtreeInitPageArg *)arg)->pBt; + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); - pPage->pPageHdr = pPage->pData; + tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); - // Init the page header - TDB_PAGE_FLAGS_SET(pPage, flags); - TDB_PAGE_NCELLS_SET(pPage, 0); - TDB_PAGE_CCELLS_SET(pPage, pBt->pageSize - sizeof(SPageFtr)); - TDB_PAGE_FCELL_SET(pPage, 0); - TDB_PAGE_NFREE_SET(pPage, 0); + if (isLeaf) { + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pData); + pLeafHdr->flags = flags; - tdbBtreeInitPage(pPage, (void *)pBt); + pPage->kLen = pBt->keyLen; + pPage->vLen = pBt->valLen; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + } else { + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pData); + pIntHdr->flags = flags; + pIntHdr->pgno = 0; + + pPage->kLen = pBt->keyLen; + pPage->vLen = sizeof(SPgno); + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + } return 0; } @@ -452,55 +452,34 @@ typedef struct { SPage *pNewPages[5]; } SBtreeBalanceHelper; -static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { - int nCells = TDB_PAGE_NCELLS(pFrom); - int cCells = TDB_PAGE_CCELLS(pFrom); - int fCell = TDB_PAGE_FCELL(pFrom); - int nFree = TDB_PAGE_NFREE(pFrom); - - pTo->pFreeStart = pTo->pCellIdx + nCells * pFrom->pPageMethods->szOffset; - memcpy(pTo->pCellIdx, pFrom->pCellIdx, nCells * pFrom->pPageMethods->szOffset); - pTo->pFreeEnd = (u8 *)pTo->pPageFtr - (u8 *)(pFrom->pPageFtr) + pFrom->pFreeEnd; - memcpy(pTo->pFreeEnd, pFrom->pFreeEnd, (u8 *)pFrom->pPageFtr - pFrom->pFreeEnd); - - TDB_PAGE_NCELLS_SET(pTo, nCells); - TDB_PAGE_CCELLS_SET(pTo, cCells); - TDB_PAGE_FCELL_SET(pTo, fCell); - TDB_PAGE_NFREE_SET(pTo, nFree); - - // TODO: update other fields - - return 0; -} - static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPager *pPager; SPage *pChild; SPgno pgnoChild; int ret; - SBtreeZeroPageArg zArg; + u8 flags; + SIntHdr *pIntHdr; + SBtreeInitPageArg zArg; + u8 leaf; pPager = pRoot->pPager; + flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Allocate a new child page - zArg.flags = TDB_BTREE_LEAF; + zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); zArg.pBt = pBt; ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg); if (ret < 0) { return -1; } - // Copy the root page content to the child page - ret = tdbBtreeCopyPageContent(pRoot, pChild); - if (ret < 0) { - return -1; + if (!leaf) { + ((SIntHdr *)pChild->pData)->pgno = ((SIntHdr *)(pRoot->pData))->pgno; } - pChild->nOverflow = pRoot->nOverflow; - for (int i = 0; i < pChild->nOverflow; i++) { - pChild->apOvfl[i] = pRoot->apOvfl[i]; - pChild->aiOvfl[i] = pRoot->aiOvfl[i]; - } + // Copy the root page content to the child page + tdbPageCopy(pRoot, pChild); // Reinitialize the root page zArg.flags = TDB_BTREE_ROOT; @@ -510,225 +489,328 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; + pIntHdr = (SIntHdr *)(pRoot->pData); + pIntHdr->pgno = pgnoChild; *ppChild = pChild; return 0; } -static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { - int nCells; - int i; - int idxStart; - int nChild; - int ret; - SPage *pParent; - SPgno pgno; - SCell *pCell; - SCellDecoder cd; - SBTree *pBt; +static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { + int ret; - pParent = pBlh->pParent; - nCells = TDB_PAGE_NCELLS(pParent); - nChild = nCells + 1; - pBt = pBlh->pBt; + int nOlds; + SPage *pOlds[3] = {0}; + SCell *pDivCell[3] = {0}; + int szDivCell[3]; + int sIdx; + u8 childNotLeaf; + SPgno rPgno; - // TODO: ASSERT below needs to be removed - ASSERT(pParent->nOverflow == 0); - ASSERT(pBlh->idx <= nCells); + { // Find 3 child pages at most to do balance + int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + SCell *pCell; - if (nChild < 3) { - idxStart = 0; - pBlh->nOld = nChild; - } else { - if (pBlh->idx == 0) { - idxStart = 0; - } else if (pBlh->idx == nCells) { - idxStart = pBlh->idx - 2; + if (nCells <= 2) { + sIdx = 0; + nOlds = nCells + 1; } else { - idxStart = pBlh->idx - 1; - } - pBlh->nOld = 3; - } - - i = pBlh->nOld - 1; - - if (idxStart + i == nCells) { - pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; - } else { - pCell = TDB_PAGE_CELL_AT(pParent, idxStart + i); - // TODO: no need to decode the payload part, and even the kLen, vLen part - // we only need the pgno part - ret = tdbBtreeDecodeCell(pParent, pCell, &cd); - if (ret < 0) { - ASSERT(0); - return -1; - } - pgno = cd.pgno; - } - for (;;) { - ret = tdbPagerFetchPage(pBt->pPager, pgno, &(pBlh->pOldPages[i]), tdbBtreeInitPage, pBt); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Loop over - if ((i--) == 0) break; - - { - // TODO - // ASSERT(0); - } - } - - return 0; -} - -static int tdbBtreeBalanceStep2(SBtreeBalanceHelper *pBlh) { -#if 0 - SPage *pPage; - int oidx; - int cidx; - int limit; - SCell *pCell; - - for (int i = 0; i < pBlh->nOld; i++) { - pPage = pBlh->pOldPages[i]; - oidx = 0; - cidx = 0; - - if (oidx < pPage->nOverflow) { - limit = pPage->aiOvfl[oidx]; - } else { - limit = pPage->pPageHdr->nCells; - } - - // Loop to copy each cell pointer out - for (;;) { - if (oidx >= pPage->nOverflow && cidx >= pPage->pPageHdr->nCells) break; - - if (cidx < limit) { - // Get local cells - pCell = TDB_PAGE_CELL_AT(pPage, cidx); - } else if (cidx == limit) { - // Get overflow cells - pCell = pPage->apOvfl[oidx++]; - - if (oidx < pPage->nOverflow) { - limit = pPage->aiOvfl[oidx]; - } else { - limit = pPage->pPageHdr->nCells; - } + // has more than three child pages + if (idx == 0) { + sIdx = 0; + } else if (idx == nCells) { + sIdx = idx - 2; } else { + sIdx = idx - 1; + } + nOlds = 3; + } + for (int i = 0; i < nOlds; i++) { + ASSERT(sIdx + i <= nCells); + + SPgno pgno; + if (sIdx + i == nCells) { + ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); + pgno = ((SIntHdr *)(pParent->pData))->pgno; + } else { + pCell = tdbPageGetCell(pParent, sIdx + i); + pgno = *(SPgno *)pCell; + } + + ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt); + if (ret < 0) { ASSERT(0); + return -1; + } + } + // copy the parent key out if child pages are not leaf page + childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); + if (childNotLeaf) { + for (int i = 0; i < nOlds; i++) { + if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) { + pCell = tdbPageGetCell(pParent, sIdx + i); + szDivCell[i] = tdbBtreeCellSize(pParent, pCell); + pDivCell[i] = malloc(szDivCell[i]); + memcpy(pDivCell[i], pCell, szDivCell[i]); + } + + if (i < nOlds - 1) { + ((SPgno *)pDivCell[i])[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; + ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); + } + } + rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; + } + // drop the cells on parent page + for (int i = 0; i < nOlds; i++) { + nCells = TDB_PAGE_TOTAL_CELLS(pParent); + if (sIdx < nCells) { + tdbPageDropCell(pParent, sIdx); + } else { + ((SIntHdr *)pParent->pData)->pgno = 0; + } + } + } + + int nNews = 0; + struct { + int cnt; + int size; + int iPage; + int oIdx; + } infoNews[5] = {0}; + + { // Get how many new pages are needed and the new distribution + + // first loop to find minimum number of pages needed + for (int oPage = 0; oPage < nOlds; oPage++) { + SPage *pPage = pOlds[oPage]; + SCell *pCell; + int cellBytes; + int oIdx; + + for (oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + pCell = tdbPageGetCell(pPage, oIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + + if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { + // page is full, use a new page + nNews++; + + ASSERT(infoNews[nNews].size + cellBytes <= TDB_PAGE_USABLE_SIZE(pPage)); + + if (childNotLeaf) { + // for non-child page, this cell is used as the right-most child, + // the divider cell to parent as well + continue; + } + } + infoNews[nNews].cnt++; + infoNews[nNews].size += cellBytes; + infoNews[nNews].iPage = oPage; + infoNews[nNews].oIdx = oIdx; } } - { - // TODO: Copy divider cells here + nNews++; + + // back loop to make the distribution even + for (int iNew = nNews - 1; iNew > 0; iNew--) { + SCell *pCell; + int szLCell, szRCell; + + for (;;) { + pCell = tdbPageGetCell(pOlds[infoNews[iNew - 1].iPage], infoNews[iNew - 1].oIdx); + + if (childNotLeaf) { + szLCell = szRCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + } else { + szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + + int iPage = infoNews[iNew - 1].iPage; + int oIdx = infoNews[iNew - 1].oIdx + 1; + SPage *pPage; + for (;;) { + pPage = pOlds[iPage]; + if (oIdx < TDB_PAGE_TOTAL_CELLS(pPage)) { + break; + } + + iPage++; + oIdx = 0; + } + + pCell = tdbPageGetCell(pPage, oIdx); + szRCell = tdbBtreeCellSize(pPage, pCell); + } + + ASSERT(infoNews[iNew - 1].cnt > 0); + + if (infoNews[iNew].size + szRCell >= infoNews[iNew - 1].size - szRCell) { + break; + } + + // Move a cell right forward + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= szLCell; + infoNews[iNew - 1].oIdx--; + for (;;) { + if (infoNews[iNew - 1].oIdx >= 0) { + break; + } + + infoNews[iNew - 1].iPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].iPage]) - 1; + } + + infoNews[iNew].cnt++; + infoNews[iNew].size += szRCell; + } } } - /* TODO */ + SPage *pNews[5] = {0}; + { // Allocate new pages, reuse the old page when possible -#endif - return 0; -} + SPgno pgno; + SBtreeInitPageArg iarg; + u8 flags; -static int tdbBtreeBalanceStep3(SBtreeBalanceHelper *pBlh) { - // Figure out number of pages needed after balance - for (int i = 0; i < pBlh->nOld; i++) { - /* TODO */ + flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); + + for (int iNew = 0; iNew < nNews; iNew++) { + if (iNew < nOlds) { + pNews[iNew] = pOlds[iNew]; + } else { + iarg.pBt = pBt; + iarg.flags = flags; + ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg); + if (ret < 0) { + ASSERT(0); + } + } + } + + // TODO: sort the page according to the page number + } + + { // Do the real cell distribution + SPage *pOldsCopy[3] = {0}; + SCell *pCell; + int szCell; + SBtreeInitPageArg iarg; + int iNew, nNewCells; + SCellDecoder cd; + + iarg.pBt = pBt; + iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); + for (int i = 0; i < nOlds; i++) { + tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], NULL, NULL); + tdbBtreeZeroPage(pOldsCopy[i], &iarg); + tdbPageCopy(pOlds[i], pOldsCopy[i]); + } + iNew = 0; + nNewCells = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); + + for (int iOld = 0; iOld < nOlds; iOld++) { + SPage *pPage; + + pPage = pOldsCopy[iOld]; + + for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + pCell = tdbPageGetCell(pPage, oIdx); + szCell = tdbBtreeCellSize(pPage, pCell); + + ASSERT(nNewCells <= infoNews[iNew].cnt); + ASSERT(iNew < nNews); + + if (nNewCells < infoNews[iNew].cnt) { + tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); + nNewCells++; + + // insert parent page + if (!childNotLeaf && nNewCells == infoNews[iNew].cnt) { + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + tdbBtreeDecodeCell(pPage, pCell, &cd); + + // TODO: pCell here may be inserted as an overflow cell, handle it + SCell *pNewCell = malloc(cd.kLen + 9); + int szNewCell; + SPgno pgno; + pgno = TDB_PAGE_PGNO(pNews[iNew]); + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); + tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); + free(pNewCell); + } + + // move to next new page + iNew++; + nNewCells = 0; + if (iNew < nNews) { + tdbBtreeZeroPage(pNews[iNew], &iarg); + } + } + } else { + ASSERT(childNotLeaf); + ASSERT(iNew < nNews - 1); + + // set current new page right-most child + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + + // insert to parent as divider cell + ASSERT(iNew < nNews - 1); + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + + // move to next new page + iNew++; + nNewCells = 0; + if (iNew < nNews) { + tdbBtreeZeroPage(pNews[iNew], &iarg); + } + } + } + } + + if (childNotLeaf) { + ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; + + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); + } else { + ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); + tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); + } + } + + for (int i = 0; i < nOlds; i++) { + tdbPageDestroy(pOldsCopy[i], NULL, NULL); + } + } + + for (int i = 0; i < 3; i++) { + if (pDivCell[i]) { + free(pDivCell[i]); + } } return 0; } -static int tdbBtreeBalanceStep4(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - -static int tdbBtreeBalanceStep5(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - -static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - -static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { - int ret; - SBtreeBalanceHelper blh; - - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); - - blh.pBt = pBt; - blh.pParent = pParent; - blh.idx = idx; - - // Step 1: find two sibling pages and get engough info about the old pages - ret = tdbBtreeBalanceStep1(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 2: Load all cells on the old page and the divider cells - ret = tdbBtreeBalanceStep2(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 3: Get the number of pages needed to hold all cells - ret = tdbBtreeBalanceStep3(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 4: Allocate enough new pages. Reuse old pages as much as possible - ret = tdbBtreeBalanceStep4(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 5: Insert new divider cells into pParent - ret = tdbBtreeBalanceStep5(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 6: Update the sibling pages - ret = tdbBtreeBalanceStep6(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - { - // TODO: Reset states - } - - { - // TODO: Clear resources - } - - return 0; -} - -static int tdbBtreeBalance(SBtCursor *pCur) { +static int tdbBtreeBalance(SBTC *pCur) { int iPage; SPage *pParent; SPage *pPage; int ret; - u16 flags; + u8 flags; u8 leaf; u8 root; @@ -736,17 +818,10 @@ static int tdbBtreeBalance(SBtCursor *pCur) { for (;;) { iPage = pCur->iPage; pPage = pCur->pPage; - flags = TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); - // TODO: Get the page free space if not get yet - // if (pPage->nFree < 0) { - // if (tdbBtreeComputeFreeSpace(pPage) < 0) { - // return -1; - // } - // } - // when the page is not overflow and not too empty, the balance work // is finished. Just break out the balance loop. if (pPage->nOverflow == 0 /* TODO: && pPage->nFree <= */) { @@ -758,7 +833,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { // ignore the case of empty if (pPage->nOverflow == 0) break; - ret = tdbBtreeBalanceDeeper(pCur->pBt, pCur->pPage, &(pCur->pgStack[1])); + ret = tdbBtreeBalanceDeeper(pCur->pBt, pPage, &(pCur->pgStack[1])); if (ret < 0) { return -1; } @@ -817,9 +892,10 @@ static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, i return 0; } +// TODO: allow vLen = 0 static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell) { - u16 flags; + u8 flags; u8 leaf; int nHeader; int nPayload; @@ -830,10 +906,18 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nPayload = 0; nHeader = 0; - flags = TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part + /* Encode SPgno if interior page */ + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; + nHeader = nHeader + sizeof(SPgno); + } + /* Encode kLen if need */ if (pPage->kLen == TDB_VARIANT_LEN) { nHeader += tdbPutVarInt(pCell + nHeader, kLen); @@ -844,14 +928,6 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nHeader += tdbPutVarInt(pCell + nHeader, vLen); } - /* Encode SPgno if interior page */ - if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); - - ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; - nHeader = nHeader + sizeof(SPgno); - } - // 2. Encode payload part if (leaf) { ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, pVal, vLen, &nPayload); @@ -893,13 +969,13 @@ static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder } static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder) { - u16 flags; + u8 flags; u8 leaf; int nHeader; int ret; nHeader = 0; - flags = TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Clear the state of decoder @@ -910,6 +986,14 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD pDecoder->pgno = 0; // 1. Decode header part + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; + pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); + nHeader = nHeader + sizeof(SPgno); + } + if (pPage->kLen == TDB_VARIANT_LEN) { nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->kLen)); } else { @@ -922,14 +1006,6 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD pDecoder->vLen = pPage->vLen; } - if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); - - pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; - pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); - nHeader = nHeader + sizeof(SPgno); - } - // 2. Decode payload part ret = tdbBtreeDecodePayload(pPage, pCell + nHeader, pDecoder); if (ret < 0) { @@ -939,4 +1015,313 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD return 0; } +static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { + u8 flags; + u8 isLeaf; + int szCell; + int kLen = 0, vLen = 0; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + szCell = 0; + + if (!isLeaf) { + szCell += sizeof(SPgno); + } + + if (pPage->kLen == TDB_VARIANT_LEN) { + szCell += tdbGetVarInt(pCell + szCell, &kLen); + } else { + kLen = pPage->kLen; + } + + if (isLeaf) { + if (pPage->vLen == TDB_VARIANT_LEN) { + szCell += tdbGetVarInt(pCell + szCell, &vLen); + } else { + vLen = pPage->vLen; + } + } + + szCell = szCell + kLen + vLen; + + return szCell; +} + +#endif + +int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { + pCur->pBt = pBt; + pCur->iPage = -1; + pCur->pPage = NULL; + pCur->idx = -1; + + return 0; +} + +int tdbBtcMoveToFirst(SBTC *pBtc) { + int ret; + SBTree *pBt; + SPager *pPager; + u8 flags; + SCell *pCell; + SPgno pgno; + + pBt = pBtc->pBt; + pPager = pBt->pPager; + + if (pBtc->iPage < 0) { + // move a clean cursor + ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->iPage = 0; + pBtc->idx = 0; + } else { + // move from a position + ASSERT(0); + } + + // move downward + for (;;) { + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + + if (TDB_BTREE_PAGE_IS_LEAF(flags)) break; + + pCell = tdbPageGetCell(pBtc->pPage, 0); + pgno = *(SPgno *)pCell; + + ret = tdbBtcMoveDownward(pBtc, pgno); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->idx = 0; + } + + return 0; +} + +int tdbBtcMoveToLast(SBTC *pBtc) { + int ret; + SBTree *pBt; + SPager *pPager; + u8 flags; + SPgno pgno; + + pBt = pBtc->pBt; + pPager = pBt->pPager; + + if (pBtc->iPage < 0) { + // move a clean cursor + ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->iPage = 0; + } else { + // move from a position + ASSERT(0); + } + + // move downward + for (;;) { + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + + if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + // TODO: handle empty case + ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0); + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1; + break; + } else { + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; + + ret = tdbBtcMoveDownward(pBtc, pgno); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + } + + return 0; +} + +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen) { + // TODO + return 0; +} + +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + SCell *pCell; + SCellDecoder cd; + void *pKey, *pVal; + int ret; + + if (pBtc->idx < 0) { + return -1; + } + + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + + tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd); + + pKey = TDB_REALLOC(*ppKey, cd.kLen); + if (pKey == NULL) { + return -1; + } + + // TODO: vLen may be zero + pVal = TDB_REALLOC(*ppVal, cd.vLen); + if (pVal == NULL) { + TDB_FREE(pKey); + return -1; + } + + *ppKey = pKey; + *ppVal = pVal; + + *kLen = cd.kLen; + *vLen = cd.vLen; + + memcpy(pKey, cd.pKey, cd.kLen); + memcpy(pVal, cd.pVal, cd.vLen); + + ret = tdbBtcMoveToNext(pBtc); + + return 0; +} + +static int tdbBtcMoveToNext(SBTC *pBtc) { + int nCells; + SPgno pgno; + SCell *pCell; + u8 flags; + + ASSERT(TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage))); + + if (pBtc->idx < 0) return -1; + + pBtc->idx++; + if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { + return 0; + } + + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } + + // Move upward + for (;;) { + tdbBtcMoveUpward(pBtc); + pBtc->idx++; + + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + if (pBtc->idx <= nCells) { + break; + } + + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } + } + + // Move downward + for (;;) { + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + if (pBtc->idx < nCells) { + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + pgno = *(SPgno *)pCell; + } else { + pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; + } + + tdbBtcMoveDownward(pBtc, pgno); + pBtc->idx = 0; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + break; + } + } + + return 0; +} + +int tdbBtcClose(SBTC *pBtc) { + // TODO + return 0; +} + +static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno) { + int ret; + + pCur->pgStack[pCur->iPage] = pCur->pPage; + pCur->idxStack[pCur->iPage] = pCur->idx; + pCur->iPage++; + pCur->pPage = NULL; + pCur->idx = -1; + + ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); + if (ret < 0) { + ASSERT(0); + } + + return 0; +} + +static int tdbBtcMoveUpward(SBTC *pBtc) { + if (pBtc->iPage == 0) return -1; + + // tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); + + pBtc->iPage--; + pBtc->pPage = pBtc->pgStack[pBtc->iPage]; + pBtc->idx = pBtc->idxStack[pBtc->iPage]; + + return 0; +} + +#ifndef NODEBUG +typedef struct { + SPgno pgno; + u8 root; + u8 leaf; + SPgno rChild; + int nCells; + int nOvfl; +} SBtPageInfo; + +SBtPageInfo btPageInfos[20]; + +void tdbBtPageInfo(SPage *pPage, int idx) { + u8 flags; + SBtPageInfo *pBtPageInfo; + + pBtPageInfo = btPageInfos + idx; + + pBtPageInfo->pgno = TDB_PAGE_PGNO(pPage); + + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + + pBtPageInfo->root = TDB_BTREE_PAGE_IS_ROOT(flags); + pBtPageInfo->leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + pBtPageInfo->rChild = 0; + if (!pBtPageInfo->leaf) { + pBtPageInfo->rChild = *(SPgno *)(pPage->pData + 1); + } + + pBtPageInfo->nCells = TDB_PAGE_TOTAL_CELLS(pPage) - pPage->nOverflow; + pBtPageInfo->nOvfl = pPage->nOverflow; +} #endif \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 00f38a19bb..4e74dc4cbb 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,13 +15,17 @@ #include "tdbInt.h" -struct STDb { +struct STDB { STEnv *pEnv; SBTree *pBt; }; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb) { - STDb *pDb; +struct STDBC { + SBTC btc; +}; + +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb) { + STDB *pDb; SPager *pPager; int ret; char fFullName[TDB_FILENAME_LEN]; @@ -30,7 +34,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF *ppDb = NULL; - pDb = (STDb *)calloc(1, sizeof(*pDb)); + pDb = (STDB *)calloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } @@ -59,23 +63,23 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF return 0; } -int tdbDbClose(STDb *pDb) { +int tdbDbClose(STDB *pDb) { // TODO return 0; } -int tdbDbDrop(STDb *pDb) { +int tdbDbDrop(STDB *pDb) { // TODO return 0; } -int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { - SBtCursor btc; - SBtCursor *pCur; - int ret; +int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { + SBTC btc; + SBTC *pCur; + int ret; pCur = &btc; - ret = tdbBtreeCursor(pCur, pDb->pBt); + ret = tdbBtcOpen(pCur, pDb->pBt); if (ret < 0) { return -1; } @@ -85,5 +89,45 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return -1; } + return 0; +} + +int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { + return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); +} + +int tdbDbcOpen(STDB *pDb, STDBC **ppDbc) { + int ret; + STDBC *pDbc = NULL; + + *ppDbc = NULL; + pDbc = malloc(sizeof(*pDbc)); + if (pDbc == NULL) { + return -1; + } + + tdbBtcOpen(&pDbc->btc, pDb->pBt); + + // TODO: move to first now, we can move to any key-value + // and in any direction, design new APIs. + ret = tdbBtcMoveToFirst(&pDbc->btc); + if (ret < 0) { + ASSERT(0); + return -1; + } + + *ppDbc = pDbc; + return 0; +} + +int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); +} + +int tdbDbcClose(STDBC *pDbc) { + if (pDbc) { + free(pDbc); + } + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 9d7181c1da..3c7d037faa 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -17,7 +17,7 @@ struct SPCache { int pageSize; int cacheSize; - TdThreadMutex mutex; + pthread_mutex_t mutex; int nFree; SPage *pFree; int nPage; @@ -53,12 +53,10 @@ static void tdbPCacheLock(SPCache *pCache); static void tdbPCacheUnlock(SPCache *pCache); static bool tdbPCacheLocked(SPCache *pCache); static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); -static void tdbPCachePinPage(SPage *pPage); -static void tdbPCacheRemovePageFromHash(SPage *pPage); -static void tdbPCacheAddPageToHash(SPage *pPage); -static void tdbPCacheUnpinPage(SPage *pPage); -static void *tdbOsMalloc(void *arg, size_t size); -static void tdbOsFree(void *arg, void *ptr); +static void tdbPCachePinPage(SPCache *pCache, SPage *pPage); +static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage); +static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage); +static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage); int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { SPCache *pCache; @@ -102,7 +100,7 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) { return pPage; } -void tdbPCacheRelease(SPage *pPage) { +void tdbPCacheRelease(SPCache *pCache, SPage *pPage) { i32 nRef; nRef = TDB_UNREF_PAGE(pPage); @@ -110,7 +108,7 @@ void tdbPCacheRelease(SPage *pPage) { if (nRef == 0) { if (1 /*TODO: page still clean*/) { - tdbPCacheUnpinPage(pPage); + tdbPCacheUnpinPage(pCache, pPage); } else { // TODO ASSERT(0); @@ -118,13 +116,13 @@ void tdbPCacheRelease(SPage *pPage) { } } -static void tdbPCacheInitLock(SPCache *pCache) { taosThreadMutexInit(&(pCache->mutex), NULL); } +static void tdbPCacheInitLock(SPCache *pCache) { pthread_mutex_init(&(pCache->mutex), NULL); } -static void tdbPCacheClearLock(SPCache *pCache) { taosThreadMutexDestroy(&(pCache->mutex)); } +static void tdbPCacheClearLock(SPCache *pCache) { pthread_mutex_destroy(&(pCache->mutex)); } -static void tdbPCacheLock(SPCache *pCache) { taosThreadMutexLock(&(pCache->mutex)); } +static void tdbPCacheLock(SPCache *pCache) { pthread_mutex_lock(&(pCache->mutex)); } -static void tdbPCacheUnlock(SPCache *pCache) { taosThreadMutexUnlock(&(pCache->mutex)); } +static void tdbPCacheUnlock(SPCache *pCache) { pthread_mutex_unlock(&(pCache->mutex)); } static bool tdbPCacheLocked(SPCache *pCache) { assert(0); @@ -144,7 +142,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe if (pPage || !alcNewPage) { if (pPage) { - tdbPCachePinPage(pPage); + tdbPCachePinPage(pCache, pPage); } return pPage; } @@ -160,8 +158,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe // 3. Try to Recycle a page if (!pPage && !pCache->lru.pLruPrev->isAnchor) { pPage = pCache->lru.pLruPrev; - tdbPCacheRemovePageFromHash(pPage); - tdbPCachePinPage(pPage); + tdbPCacheRemovePageFromHash(pCache, pPage); + tdbPCachePinPage(pCache, pPage); } // 4. Try a stress allocation (TODO) @@ -173,16 +171,13 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); pPage->pLruNext = NULL; pPage->pPager = NULL; - tdbPCacheAddPageToHash(pPage); + tdbPCacheAddPageToHash(pCache, pPage); } return pPage; } -static void tdbPCachePinPage(SPage *pPage) { - SPCache *pCache; - - pCache = pPage->pCache; +static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { if (!PAGE_IS_PINNED(pPage)) { pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; @@ -192,11 +187,8 @@ static void tdbPCachePinPage(SPage *pPage) { } } -static void tdbPCacheUnpinPage(SPage *pPage) { - SPCache *pCache; - i32 nRef; - - pCache = pPage->pCache; +static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { + i32 nRef; tdbPCacheLock(pCache); @@ -217,12 +209,10 @@ static void tdbPCacheUnpinPage(SPage *pPage) { tdbPCacheUnlock(pCache); } -static void tdbPCacheRemovePageFromHash(SPage *pPage) { - SPCache *pCache; - SPage **ppPage; - int h; +static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { + SPage **ppPage; + int h; - pCache = pPage->pCache; h = PCACHE_PAGE_HASH(&(pPage->pgid)); for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) ; @@ -232,11 +222,9 @@ static void tdbPCacheRemovePageFromHash(SPage *pPage) { pCache->nPage--; } -static void tdbPCacheAddPageToHash(SPage *pPage) { - SPCache *pCache; - int h; +static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { + int h; - pCache = pPage->pCache; h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash; pPage->pHashNext = pCache->pgHash[h]; @@ -257,7 +245,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pCache->nFree = 0; pCache->pFree = NULL; for (int i = 0; i < pCache->cacheSize; i++) { - ret = tdbPageCreate(pCache->pageSize, &pPage, tdbOsMalloc, NULL); + ret = tdbPageCreate(pCache->pageSize, &pPage, NULL, NULL); if (ret < 0) { // TODO: handle error return -1; @@ -266,7 +254,6 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { // pPage->pgid = 0; pPage->isAnchor = 0; pPage->isLocalPage = 1; - pPage->pCache = pCache; TDB_INIT_PAGE_REF(pPage); pPage->pHashNext = NULL; pPage->pLruNext = NULL; @@ -297,13 +284,3 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { } int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; } - -static void *tdbOsMalloc(void *arg, size_t size) { - void *ptr; - - ptr = malloc(size); - - return ptr; -} - -static void tdbOsFree(void *arg, void *ptr) { free(ptr); } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c deleted file mode 100644 index df158de756..0000000000 --- a/source/libs/tdb/src/db/tdbPage.c +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -typedef struct __attribute__((__packed__)) { - u8 szCell[2]; - u8 nxOffset[2]; -} SFreeCell; - -typedef struct __attribute__((__packed__)) { - u8 szCell[3]; - u8 nxOffset[3]; -} SFreeCellL; - -/* For small page */ -#define TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCell *)(PCELL))->szCell) -#define TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCell *)(PCELL))->nxOffset) - -#define TDB_SPAGE_FREE_CELL_SIZE(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))[0] -#define TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))[0] - -#define TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) (TDB_SPAGE_FREE_CELL_SIZE(PCELL) = (SIZE)) -#define TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) (TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) = (OFFSET)) - -/* For large page */ -#define TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCellL *)(PCELL))->szCell) -#define TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCellL *)(PCELL))->nxOffset) - -#define TDB_LPAGE_FREE_CELL_SIZE(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL)) -#define TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) - -#define TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL), SIZE) -#define TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL), OFFSET) - -/* For page */ -#define TDB_PAGE_FREE_CELL_SIZE_PTR(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) : TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL)) -#define TDB_PAGE_FREE_CELL_NXOFFSET_PTR(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) - -#define TDB_PAGE_FREE_CELL_SIZE(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE(PCELL) : TDB_SPAGE_FREE_CELL_SIZE(PCELL)) -#define TDB_PAGE_FREE_CELL_NXOFFSET(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL)) - -#define TDB_PAGE_FREE_CELL_SIZE_SET(PPAGE, PCELL, SIZE) \ - do { \ - if (TDB_IS_LARGE_PAGE(PPAGE)) { \ - TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ - } else { \ - TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ - } \ - } while (0) -#define TDB_PAGE_FREE_CELL_NXOFFSET_SET(PPAGE, PCELL, OFFSET) \ - do { \ - if (TDB_IS_LARGE_PAGE(PPAGE)) { \ - TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ - } else { \ - TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ - } \ - } while (0) - -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); -static int tdbPageDefragment(SPage *pPage); - -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { - SPage *pPage; - u8 *ptr; - int size; - - ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); - - *ppPage = NULL; - size = pageSize + sizeof(*pPage); - - ptr = (u8 *)((*xMalloc)(arg, size)); - if (pPage == NULL) { - return -1; - } - - memset(ptr, 0, size); - pPage = (SPage *)(ptr + pageSize); - - pPage->pData = ptr; - pPage->pageSize = pageSize; - if (pageSize < 65536) { - pPage->szOffset = 2; - pPage->szPageHdr = sizeof(SPageHdr); - pPage->szFreeCell = sizeof(SFreeCell); - } else { - pPage->szOffset = 3; - pPage->szPageHdr = sizeof(SPageHdrL); - pPage->szFreeCell = sizeof(SFreeCellL); - } - TDB_INIT_PAGE_LOCK(pPage); - - /* TODO */ - - *ppPage = pPage; - return 0; -} - -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { - u8 *ptr; - - ptr = pPage->pData; - (*xFree)(arg, ptr); - - return 0; -} - -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { - int ret; - SCell *pTarget; - u8 *pTmp; - int j; - - if (pPage->nOverflow || szCell + pPage->szOffset > pPage->nFree) { - // TODO: need to figure out if pCell may be used by outside of this function - j = pPage->nOverflow++; - - pPage->apOvfl[j] = pCell; - pPage->aiOvfl[j] = idx; - } else { - ret = tdbPageAllocate(pPage, szCell, &pTarget); - if (ret < 0) { - return -1; - } - - memcpy(pTarget, pCell, szCell); - pTmp = pPage->pCellIdx + idx * pPage->szOffset; - memmove(pTmp + pPage->szOffset, pTmp, pPage->pFreeStart - pTmp - pPage->szOffset); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); - TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); - } - - return 0; -} - -int tdbPageDropCell(SPage *pPage, int idx) { - // TODO - return 0; -} - -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { - SCell *pCell; - SFreeCell *pFreeCell; - u8 *pOffset; - int ret; - - ASSERT(pPage->nFree > size + pPage->szOffset); - - pCell = NULL; - *ppCell = NULL; - - // 1. Try to allocate from the free space area - if (pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset) { - pPage->pFreeEnd -= size; - pPage->pFreeStart += pPage->szOffset; - pCell = pPage->pFreeEnd; - } - - // 2. Try to allocate from the page free list - if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= pPage->szOffset) && TDB_PAGE_FCELL(pPage)) { - int szCell; - int nxOffset; - - pCell = pPage->pData + TDB_PAGE_FCELL(pPage); - pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell - : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - - for (;;) { - // Find a cell - if (szCell >= size) { - if (szCell - size >= pPage->szFreeCell) { - SCell *pTmpCell = pCell + size; - - TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); - TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); - // TODO: *pOffset = pTmpCell - pPage->pData; - } else { - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); - // TODO: *pOffset = nxOffset; - } - break; - } - - // Not find a cell yet - if (nxOffset > 0) { - pCell = pPage->pData + nxOffset; - pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - continue; - } else { - pCell = NULL; - break; - } - } - - if (pCell) { - pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; - } - } - - // 3. Try to dfragment and allocate again - if (pCell == NULL) { - ret = tdbPageDefragment(pPage); - if (ret < 0) { - return -1; - } - - ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset); - ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); - - // Allocate from the free space area again - pPage->pFreeEnd -= size; - pPage->pFreeStart += pPage->szOffset; - pCell = pPage->pFreeEnd; - } - - ASSERT(pCell != NULL); - - pPage->nFree = pPage->nFree - size - pPage->szOffset; - *ppCell = pCell; - return 0; -} - -static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { - // TODO - return 0; -} - -static int tdbPageDefragment(SPage *pPage) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ac0bc15e1d..fe4b9aa123 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -255,6 +255,10 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage return 0; } +void tdbPagerReturnPage(SPager *pPager, SPage *pPage) { + tdbPCacheRelease(pPager->pCache, pPage); +} + static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { // TODO: Allocate a page from the free list return 0; diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index c1fe77c22e..e46e8bb78b 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -20,10 +20,15 @@ extern "C" { #endif -typedef struct SBTree SBTree; -typedef struct SBtCursor SBtCursor; +typedef struct SBTree SBTree; +typedef struct SBTC SBTC; +typedef struct SBtInfo { + SPgno root; + int nLevel; + int nData; +} SBtInfo; -struct SBtCursor { +struct SBTC { SBTree *pBt; i8 iPage; SPage *pPage; @@ -33,10 +38,19 @@ struct SBtCursor { void *pBuf; }; +// SBTree int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); -int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt); -int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); + +// SBTC +int tdbBtcOpen(SBTC *pCur, SBTree *pBt); +int tdbBtcMoveToFirst(SBTC *pBtc); +int tdbBtcMoveToLast(SBTC *pBtc); +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen); +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbBtcClose(SBTC *pBtc); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 06ea74a83e..b96076b826 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -20,12 +20,20 @@ extern "C" { #endif -typedef struct STDb STDb; +typedef struct STDB STDB; +typedef struct STDBC STDBC; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb); -int tdbDbClose(STDb *pDb); -int tdbDbDrop(STDb *pDb); -int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); +// STDB +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb); +int tdbDbClose(STDB *pDb); +int tdbDbDrop(STDB *pDb); +int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); +int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); + +// STDBC +int tdbDbcOpen(STDB *pDb, STDBC **ppDbc); +int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbDbcClose(STDBC *pDbc); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 5902a6a716..98845bb66f 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -95,7 +95,7 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { // tdb_log #define tdbError(var) -typedef TD_DLIST(STDb) STDbList; +typedef TD_DLIST(STDB) STDbList; typedef TD_DLIST(SPgFile) SPgFileList; typedef TD_DLIST_NODE(SPgFile) SPgFileListNode; @@ -141,8 +141,8 @@ typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, i #define TDB_FLAG_IS(flags, flag) ((flags) == (flag)) #define TDB_FLAG_HAS(flags, flag) (((flags) & (flag)) != 0) #define TDB_FLAG_NO(flags, flag) ((flags) & (flag) == 0) -#define TDB_FLAG_ADD(flags, flag) ((flags) |= (flag)) -#define TDB_FLAG_REMOVE(flags, flag) ((flags) &= (~(flag))) +#define TDB_FLAG_ADD(flags, flag) ((flags) | (flag)) +#define TDB_FLAG_REMOVE(flags, flag) ((flags) & (~(flag))) typedef struct SPager SPager; typedef struct SPCache SPCache; diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h index ff4f1acbb6..c7fa155615 100644 --- a/source/libs/tdb/src/inc/tdbPCache.h +++ b/source/libs/tdb/src/inc/tdbPCache.h @@ -21,23 +21,22 @@ extern "C" { #endif #define TDB_PCACHE_PAGE \ - u8 isAnchor; \ - u8 isLocalPage; \ - u8 isDirty; \ - i32 nRef; \ - SPCache *pCache; \ - SPage *pFreeNext; \ - SPage *pHashNext; \ - SPage *pLruNext; \ - SPage *pLruPrev; \ - SPage *pDirtyNext; \ - SPager *pPager; \ - SPgid pgid; + u8 isAnchor; \ + u8 isLocalPage; \ + u8 isDirty; \ + i32 nRef; \ + SPage *pFreeNext; \ + SPage *pHashNext; \ + SPage *pLruNext; \ + SPage *pLruPrev; \ + SPage *pDirtyNext; \ + SPager *pPager; \ + SPgid pgid; int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); int tdbPCacheClose(SPCache *pCache); SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); -void tdbPCacheRelease(SPage *pPage); +void tdbPCacheRelease(SPCache *pCache, SPage *pPage); int tdbPCacheGetPageSize(SPCache *pCache); #ifdef __cplusplus diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 8a51c331b6..a6f9fbf615 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -27,9 +27,6 @@ typedef struct { int szOffset; int szPageHdr; int szFreeCell; - // flags - u16 (*getFlags)(SPage *); - void (*setFlags)(SPage *, u16); // cell number int (*getCellNum)(SPage *); void (*setCellNum)(SPage *, int); @@ -45,6 +42,9 @@ typedef struct { // cell offset at idx int (*getCellOffset)(SPage *, int); void (*setCellOffset)(SPage *, int, int); + // free cell info + void (*getFreeCellInfo)(SCell *pCell, int *szCell, int *nxOffset); + void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); } SPageMethods; // Page footer @@ -53,58 +53,37 @@ typedef struct __attribute__((__packed__)) { } SPageFtr; struct SPage { - TdThreadSpinlock lock; - u8 *pData; + pthread_spinlock_t lock; int pageSize; + u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am - u8 szAmHdr; u8 *pPageHdr; - u8 *pAmHdr; u8 *pCellIdx; u8 *pFreeStart; u8 *pFreeEnd; SPageFtr *pPageFtr; - int kLen; // key length of the page, -1 for unknown - int vLen; // value length of the page, -1 for unknown - int nFree; - int maxLocal; - int minLocal; int nOverflow; SCell *apOvfl[4]; int aiOvfl[4]; + int kLen; // key length of the page, -1 for unknown + int vLen; // value length of the page, -1 for unknown + int maxLocal; + int minLocal; + int (*xCellSize)(const SPage *, SCell *); // Fields used by SPCache TDB_PCACHE_PAGE }; -/* For page */ -#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) -#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) -#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) -#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) -#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) -#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) - -#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) -#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) -#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) -#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) -#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) -#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) - -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) - -#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) - // For page lock #define P_LOCK_SUCC 0 #define P_LOCK_BUSY 1 #define P_LOCK_FAIL -1 -#define TDB_INIT_PAGE_LOCK(pPage) taosThreadSpinInit(&((pPage)->lock), 0) -#define TDB_DESTROY_PAGE_LOCK(pPage) taosThreadSpinDestroy(&((pPage)->lock)) -#define TDB_LOCK_PAGE(pPage) taosThreadSpinLock(&((pPage)->lock)) -#define TDB_UNLOCK_PAGE(pPage) taosThreadSpinUnlock(&((pPage)->lock)) +#define TDB_INIT_PAGE_LOCK(pPage) pthread_spin_init(&((pPage)->lock), 0) +#define TDB_DESTROY_PAGE_LOCK(pPage) pthread_spin_destroy(&((pPage)->lock)) +#define TDB_LOCK_PAGE(pPage) pthread_spin_lock(&((pPage)->lock)) +#define TDB_UNLOCK_PAGE(pPage) pthread_spin_unlock(&((pPage)->lock)) #define TDB_TRY_LOCK_PAGE(pPage) \ ({ \ int ret; \ @@ -119,10 +98,43 @@ struct SPage { }) // APIs -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); -int tdbPageDropCell(SPage *pPage, int idx); +#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) +#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) +#define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) + +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); +int tdbPageDropCell(SPage *pPage, int idx); +void tdbPageCopy(SPage *pFromPage, SPage *pToPage); + +static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { + SCell *pCell; + int iOvfl; + int lidx; + + ASSERT(idx >= 0 && idx < TDB_PAGE_TOTAL_CELLS(pPage)); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + pCell = pPage->apOvfl[iOvfl]; + return pCell; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } + } + + lidx = idx - iOvfl; + ASSERT(lidx >= 0 && lidx < pPage->pPageMethods->getCellNum(pPage)); + pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); + + return pCell; +} #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h index e4ed8552fd..f4cc822f27 100644 --- a/source/libs/tdb/src/inc/tdbPager.h +++ b/source/libs/tdb/src/inc/tdbPager.h @@ -20,15 +20,16 @@ extern "C" { #endif -int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); -int tdbPagerClose(SPager *pPager); -int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); -int tdbPagerWrite(SPager *pPager, SPage *pPage); -int tdbPagerBegin(SPager *pPager); -int tdbPagerCommit(SPager *pPager); -int tdbPagerGetPageSize(SPager *pPager); -int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); -int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); +int tdbPagerClose(SPager *pPager); +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); +int tdbPagerWrite(SPager *pPager, SPage *pPage); +int tdbPagerBegin(SPager *pPager); +int tdbPagerCommit(SPager *pPager); +int tdbPagerGetPageSize(SPager *pPager); +int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +void tdbPagerReturnPage(SPager *pPager, SPage *pPage); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 8aaded933a..30ad02db82 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -39,6 +39,38 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); int tdbPRead(int fd, void *pData, int count, i64 offset); +#define TDB_REALLOC(PTR, SIZE) \ + ({ \ + void *nPtr; \ + if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ + nPtr = realloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ + if (nPtr) { \ + ((int *)nPtr)[0] = (SIZE); \ + nPtr = (char *)nPtr + sizeof(int); \ + } \ + } else { \ + nPtr = (PTR); \ + } \ + nPtr; \ + }) + +#define TDB_FREE(PTR) \ + do { \ + if (PTR) { \ + free((char *)(PTR) - sizeof(int)); \ + } \ + } while (0) + +static inline void *tdbOsMalloc(void *arg, size_t size) { + void *ptr; + + ptr = malloc(size); + + return ptr; +} + +static inline void tdbOsFree(void *arg, void *ptr) { free(ptr); } + static inline int tdbPutVarInt(u8 *p, int v) { int n = 0; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4ec3a895e7..516330e4e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,13 +18,25 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -typedef struct __attribute__((__packed__)) { - u16 szCell; - u16 nxOffset; -} SFreeCell; +#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) +#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) +#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) +#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) +#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) +#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) +#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) +#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) +#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) +#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) +#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) +#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) +#define TDB_PAGE_MAX_FREE_BLOCK(pPage, szAmHdr) \ + ((pPage)->pageSize - (szAmHdr)-TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { SPage *pPage; @@ -35,25 +47,26 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = NULL; size = pageSize + sizeof(*pPage); + if (xMalloc == NULL) { + xMalloc = tdbOsMalloc; + } ptr = (u8 *)((*xMalloc)(arg, size)); - if (pPage == NULL) { + if (ptr == NULL) { return -1; } memset(ptr, 0, size); pPage = (SPage *)(ptr + pageSize); - pPage->pData = ptr; + TDB_INIT_PAGE_LOCK(pPage); pPage->pageSize = pageSize; + pPage->pData = ptr; if (pageSize < 65536) { pPage->pPageMethods = &pageMethods; } else { pPage->pPageMethods = &pageLargeMethods; } - TDB_INIT_PAGE_LOCK(pPage); - - /* TODO */ *ppPage = pPage; return 0; @@ -62,157 +75,365 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; + if (!xFree) { + xFree = tdbOsFree; + } + ptr = pPage->pData; (*xFree)(arg, ptr); return 0; } -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { - int ret; - SCell *pTarget; - u8 *pTmp; - int j; +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { + pPage->pPageHdr = pPage->pData + szAmHdr; + TDB_PAGE_NCELLS_SET(pPage, 0); + TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); + TDB_PAGE_FCELL_SET(pPage, 0); + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage, szAmHdr)); + pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); + pPage->pFreeStart = pPage->pCellIdx; + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->nOverflow = 0; + pPage->xCellSize = xCellSize; - if (pPage->nOverflow || szCell + TDB_PAGE_OFFSET_SIZE(pPage) > pPage->nFree) { - // TODO: need to figure out if pCell may be used by outside of this function - j = pPage->nOverflow++; + ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); +} - pPage->apOvfl[j] = pCell; - pPage->aiOvfl[j] = idx; - } else { - ret = tdbPageAllocate(pPage, szCell, &pTarget); - if (ret < 0) { - return -1; +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { + pPage->pPageHdr = pPage->pData + szAmHdr; + pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); + pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->nOverflow = 0; + pPage->xCellSize = xCellSize; + + ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); + ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); +} + +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl) { + int nFree; + int nCells; + int iOvfl; + int lidx; // local idx + SCell *pNewCell; + + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); + + nFree = TDB_PAGE_NFREE(pPage); + nCells = TDB_PAGE_NCELLS(pPage); + iOvfl = 0; + + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] >= idx) { + break; + } + } + + lidx = idx - iOvfl; + + if (asOvfl || nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + // TODO: make it extensible + // add the cell as an overflow cell + for (int i = pPage->nOverflow; i > iOvfl; i--) { + pPage->apOvfl[i] = pPage->apOvfl[i - 1]; + pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; } - memcpy(pTarget, pCell, szCell); - pTmp = pPage->pCellIdx + idx * TDB_PAGE_OFFSET_SIZE(pPage); - memmove(pTmp + TDB_PAGE_OFFSET_SIZE(pPage), pTmp, pPage->pFreeStart - pTmp - TDB_PAGE_OFFSET_SIZE(pPage)); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); - TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); + // TODO: here has memory leak + pNewCell = (SCell *)malloc(szCell); + memcpy(pNewCell, pCell, szCell); + + pPage->apOvfl[iOvfl] = pNewCell; + pPage->aiOvfl[iOvfl] = idx; + pPage->nOverflow++; + iOvfl++; + } else { + // page must has enough space to hold the cell locally + tdbPageAllocate(pPage, szCell, &pNewCell); + + memcpy(pNewCell, pCell, szCell); + + // no overflow cell exists in this page + u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; + u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); + memmove(dest, src, pPage->pFreeStart - dest); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, pNewCell - pPage->pData); + TDB_PAGE_NCELLS_SET(pPage, nCells + 1); + + ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); + } + + for (; iOvfl < pPage->nOverflow; iOvfl++) { + pPage->aiOvfl[iOvfl]++; } return 0; } int tdbPageDropCell(SPage *pPage, int idx) { - // TODO + int lidx; + SCell *pCell; + int szCell; + int nCells; + int iOvfl; + + nCells = TDB_PAGE_NCELLS(pPage); + + ASSERT(idx >= 0 && idx < nCells + pPage->nOverflow); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + // remove the over flow cell + for (; (++iOvfl) < pPage->nOverflow;) { + pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1; + pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl]; + } + + pPage->nOverflow--; + return 0; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } + } + + lidx = idx - iOvfl; + pCell = TDB_PAGE_CELL_AT(pPage, lidx); + szCell = (*pPage->xCellSize)(pPage, pCell); + tdbPageFree(pPage, lidx, pCell, szCell); + TDB_PAGE_NCELLS_SET(pPage, nCells - 1); + + for (; iOvfl < pPage->nOverflow; iOvfl++) { + pPage->aiOvfl[iOvfl]--; + ASSERT(pPage->aiOvfl[iOvfl] > 0); + } + return 0; } -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { - SCell *pCell; - SFreeCell *pFreeCell; - u8 *pOffset; - int ret; +void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { + int delta, nFree; - ASSERT(pPage->nFree > size + TDB_PAGE_OFFSET_SIZE(pPage)); + pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); + pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); + + ASSERT(pToPage->pFreeEnd >= pToPage->pFreeStart); + + memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); + memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); + + ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); + + delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); + if (delta != 0) { + nFree = TDB_PAGE_NFREE(pFromPage); + TDB_PAGE_NFREE_SET(pToPage, nFree - delta); + } + + // Copy the overflow cells + for (int iOvfl = 0; iOvfl < pFromPage->nOverflow; iOvfl++) { + pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl]; + pToPage->apOvfl[iOvfl] = pFromPage->apOvfl[iOvfl]; + } + pToPage->nOverflow = pFromPage->nOverflow; +} + +static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { + SCell *pFreeCell; + u8 *pOffset; + int nFree; + int ret; + int cellFree; + SCell *pCell = NULL; - pCell = NULL; *ppCell = NULL; + nFree = TDB_PAGE_NFREE(pPage); - // 1. Try to allocate from the free space area - if (pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)) { - pPage->pFreeEnd -= size; - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + ASSERT(nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)); + ASSERT(TDB_PAGE_CCELLS(pPage) == pPage->pFreeEnd - pPage->pData); + + // 1. Try to allocate from the free space block area + if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + pPage->pFreeEnd -= szCell; pCell = pPage->pFreeEnd; + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + goto _alloc_finish; } // 2. Try to allocate from the page free list - if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) && - TDB_PAGE_FCELL(pPage)) { -#if 0 - int szCell; - int nxOffset; - - pCell = pPage->pData + TDB_PAGE_FCELL(pPage); - pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell - : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + cellFree = TDB_PAGE_FCELL(pPage); + ASSERT(cellFree == 0 || cellFree > pPage->pFreeEnd - pPage->pData); + if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { + SCell *pPrevFreeCell = NULL; + int szPrevFreeCell; + int szFreeCell; + int nxFreeCell; + int newSize; for (;;) { - // Find a cell - if (szCell >= size) { - if (szCell - size >= pPage->szFreeCell) { - SCell *pTmpCell = pCell + size; + if (cellFree == 0) break; - TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); - TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); - // TODO: *pOffset = pTmpCell - pPage->pData; + pFreeCell = pPage->pData + cellFree; + pPage->pPageMethods->getFreeCellInfo(pFreeCell, &szFreeCell, &nxFreeCell); + + if (szFreeCell >= szCell) { + pCell = pFreeCell; + + newSize = szFreeCell - szCell; + pFreeCell += szCell; + if (newSize >= TDB_PAGE_FREE_CELL_SIZE(pPage)) { + pPage->pPageMethods->setFreeCellInfo(pFreeCell, newSize, nxFreeCell); + if (pPrevFreeCell) { + pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, pFreeCell - pPage->pData); + } else { + TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); + } } else { - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); - // TODO: *pOffset = nxOffset; + if (pPrevFreeCell) { + pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); + } else { + TDB_PAGE_FCELL_SET(pPage, nxFreeCell); + } } - break; - } - // Not find a cell yet - if (nxOffset > 0) { - pCell = pPage->pData + nxOffset; - pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - continue; + goto _alloc_finish; } else { - pCell = NULL; - break; + pPrevFreeCell = pFreeCell; + szPrevFreeCell = szFreeCell; + cellFree = nxFreeCell; } } - - if (pCell) { - pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; - } -#endif } // 3. Try to dfragment and allocate again - if (pCell == NULL) { - ret = tdbPageDefragment(pPage); - if (ret < 0) { - return -1; - } + tdbPageDefragment(pPage); + ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + ASSERT(nFree == TDB_PAGE_NFREE(pPage)); + ASSERT(pPage->pFreeEnd - pPage->pData == TDB_PAGE_CCELLS(pPage)); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)); - ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + pPage->pFreeEnd -= szCell; + pCell = pPage->pFreeEnd; + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); - // Allocate from the free space area again - pPage->pFreeEnd -= size; - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - pCell = pPage->pFreeEnd; - } - - ASSERT(pCell != NULL); - - pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); +_alloc_finish: + ASSERT(pCell); + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); *ppCell = pCell; return 0; } -static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { - // TODO +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { + int nFree; + int cellFree; + u8 *dest; + u8 *src; + + ASSERT(pCell >= pPage->pFreeEnd); + ASSERT(pCell + szCell <= (u8 *)(pPage->pPageFtr)); + ASSERT(pCell == TDB_PAGE_CELL_AT(pPage, idx)); + + nFree = TDB_PAGE_NFREE(pPage); + + if (pCell == pPage->pFreeEnd) { + pPage->pFreeEnd += szCell; + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + } else { + if (szCell >= TDB_PAGE_FREE_CELL_SIZE(pPage)) { + cellFree = TDB_PAGE_FCELL(pPage); + pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree); + TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); + } else { + ASSERT(0); + } + } + + dest = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * idx; + src = dest + TDB_PAGE_OFFSET_SIZE(pPage); + memmove(dest, src, pPage->pFreeStart - src); + + pPage->pFreeStart -= TDB_PAGE_OFFSET_SIZE(pPage); + nFree = nFree + szCell + TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree); return 0; } static int tdbPageDefragment(SPage *pPage) { - // TODO - ASSERT(0); + int nFree; + int nCells; + SCell *pCell; + SCell *pNextCell; + SCell *pTCell; + int szCell; + int idx; + int iCell; + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart < nFree); + + nFree = TDB_PAGE_NFREE(pPage); + nCells = TDB_PAGE_NCELLS(pPage); + + // Loop to compact the page content + // Here we use an O(n^2) algorithm to do the job since + // this is a low frequency job. + pNextCell = (u8 *)pPage->pPageFtr; + pCell = NULL; + for (iCell = 0;; iCell++) { + // compact over + if (iCell == nCells) { + pPage->pFreeEnd = pNextCell; + break; + } + + for (int i = 0; i < nCells; i++) { + if (TDB_PAGE_CELL_OFFSET_AT(pPage, i) < pNextCell - pPage->pData) { + pTCell = TDB_PAGE_CELL_AT(pPage, i); + if (pCell == NULL || pCell < pTCell) { + pCell = pTCell; + idx = i; + } + } else { + continue; + } + } + + ASSERT(pCell != NULL); + + szCell = (*pPage->xCellSize)(pPage, pCell); + + ASSERT(pCell + szCell <= pNextCell); + if (pCell + szCell < pNextCell) { + memmove(pNextCell - szCell, pCell, szCell); + } + + pCell = NULL; + pNextCell = pNextCell - szCell; + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pNextCell - pPage->pData); + } + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + TDB_PAGE_FCELL_SET(pPage, 0); + return 0; } /* ---------------------------------------------------------------------------------------------------------- */ typedef struct __attribute__((__packed__)) { - u16 flags; u16 cellNum; u16 cellBody; u16 cellFree; u16 nFree; } SPageHdr; -// flags -static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } -static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } +typedef struct __attribute__((__packed__)) { + u16 szCell; + u16 nxOffset; +} SFreeCell; // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } @@ -253,20 +474,33 @@ static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; } +// free cell info +static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCell *pFreeCell = (SFreeCell *)pCell; + *szCell = pFreeCell->szCell; + *nxOffset = pFreeCell->nxOffset; +} + +static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCell *pFreeCell = (SFreeCell *)pCell; + pFreeCell->szCell = szCell; + pFreeCell->nxOffset = nxOffset; +} + SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset // setCellOffset + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset, // setCellOffset + getPageFreeCellInfo, // getFreeCellInfo + setPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index e7c60118d2..c5d4a6047f 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -16,11 +16,10 @@ #include "tdbInt.h" typedef struct __attribute__((__packed__)) { - u16 flags; - u8 cellNum[3]; - u8 cellBody[3]; - u8 cellFree[3]; - u8 nFree[3]; + u8 cellNum[3]; + u8 cellBody[3]; + u8 cellFree[3]; + u8 nFree[3]; } SPageHdrL; typedef struct __attribute__((__packed__)) { @@ -28,10 +27,6 @@ typedef struct __attribute__((__packed__)) { u8 nxOffset[3]; } SFreeCellL; -// flags -static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; } -static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; } - // cellNum static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } static inline void setPageCellNum(SPage *pPage, int cellNum) { @@ -66,20 +61,33 @@ static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); } +// free cell info +static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + *szCell = TDB_GET_U24(pFreeCell->szCell); + *nxOffset = TDB_GET_U24(pFreeCell->nxOffset); +} + +static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + TDB_PUT_U24(pFreeCell->szCell, szCell); + TDB_PUT_U24(pFreeCell->nxOffset, nxOffset); +} + SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset // setCellOffset + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset, // setCellOffset + getPageFreeCellInfo, // getFreeCellInfo + setPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index c3cc922f32..e249f98f46 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -2,28 +2,190 @@ #include "tdbInt.h" +#include + +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)malloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void closePool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + free(pMem); + } while (1); + + assert(pPool->size == 0); + + free(pPool); +} + +static void *poolMalloc(void *arg, int size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)malloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + free(pMem); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} + TEST(tdb_test, simple_test) { - int ret; - STEnv *pEnv; - STDb *pDb; + int ret; + STEnv *pEnv; + STDB *pDb; + FKeyComparator compFunc; + int nData = 10000000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 20, &pEnv); + ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb); + compFunc = tKeyCmpr; + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); - { // Insert some data + { char key[64]; char val[64]; - for (int i = 1; i <= 1000; i++) { - sprintf(key, "key%d", i); - sprintf(val, "value%d", i); - ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); + { // Insert some data + + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); + GTEST_ASSERT_EQ(ret, 0); + } + } + + { // Query the data + void *pVal = NULL; + int vLen; + + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + + ret = tdbDbGet(pDb, key, strlen(key), &pVal, &vLen); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, strlen(val)); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + } + + TDB_FREE(pVal); + } + + { // Iterate to query the DB data + STDBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + + ret = tdbDbcOpen(pDb, &pDBC); GTEST_ASSERT_EQ(ret, 0); + + for (;;) { + ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; + + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbDbcClose(pDBC); + + TDB_FREE(pKey); + TDB_FREE(pVal); } } diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index bfe7904a63..e50045d30b 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -30,7 +30,7 @@ STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) { return NULL; } - STfs *pTfs = calloc(1, sizeof(STfs)); + STfs *pTfs = taosMemoryCalloc(1, sizeof(STfs)); if (pTfs == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -86,7 +86,7 @@ void tfsClose(STfs *pTfs) { taosHashCleanup(pTfs->hash); taosThreadSpinDestroy(&pTfs->lock); - free(pTfs); + taosMemoryFree(pTfs); } void tfsUpdateSize(STfs *pTfs) { @@ -184,7 +184,7 @@ void *tfsDecodeFile(STfs *pTfs, void *buf, STfsFile *pFile) { tfsInitFile(pTfs, pFile, diskId, rname); - tfree(rname); + taosMemoryFreeClear(rname); return buf; } @@ -242,12 +242,12 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { char *dir = strdup(taosDirName(s)); if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) { - free(s); - free(dir); + taosMemoryFree(s); + taosMemoryFree(dir); return -1; } - free(s); - free(dir); + taosMemoryFree(s); + taosMemoryFree(dir); if (tfsMkdirAt(pTfs, rname, diskId) < 0) { return -1; @@ -311,7 +311,7 @@ int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { } STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { - STfsDir *pDir = calloc(1, sizeof(STfsDir)); + STfsDir *pDir = taosMemoryCalloc(1, sizeof(STfsDir)); if (pDir == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -323,7 +323,7 @@ STfsDir *tfsOpendir(STfs *pTfs, const char *rname) { tstrncpy(pDir->dirname, rname, TSDB_FILENAME_LEN); if (tfsOpendirImpl(pTfs, pDir) < 0) { - free(pDir); + taosMemoryFree(pDir); return NULL; } @@ -369,7 +369,7 @@ void tfsClosedir(STfsDir *pTfsDir) { taosCloseDir(pTfsDir->pDir); pTfsDir->pDir = NULL; } - free(pTfsDir); + taosMemoryFree(pTfsDir); } } diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index 52396db3be..ff40529ab2 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -17,7 +17,7 @@ #include "tfsInt.h" STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { - STfsDisk *pDisk = calloc(1, sizeof(STfsDisk)); + STfsDisk *pDisk = taosMemoryCalloc(1, sizeof(STfsDisk)); if (pDisk == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -25,7 +25,7 @@ STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { pDisk->path = strdup(path); if (pDisk->path == NULL) { - free(pDisk); + taosMemoryFree(pDisk); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -38,8 +38,8 @@ STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { STfsDisk *tfsFreeDisk(STfsDisk *pDisk) { if (pDisk != NULL) { - free(pDisk->path); - free(pDisk); + taosMemoryFree(pDisk->path); + taosMemoryFree(pDisk); } return NULL; diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 9cb914a670..1a093c3877 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -196,7 +196,7 @@ TEST_F(TfsTest, 04_File) { { int32_t size = 1024; - void *ret = malloc(size + sizeof(size_t)); + void *ret = taosMemoryMalloc(size + sizeof(size_t)); *(size_t *)ret = size; void *buf = (void *)((char *)ret + sizeof(size_t)); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 8cfde8267d..1b8cbb0f04 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -158,7 +158,8 @@ typedef struct { char secured : 2; char spi : 2; - uint32_t code; // del later + uint64_t ahandle; // ahandle assigned by client + uint32_t code; // del later uint32_t msgType; int32_t msgLen; uint8_t content[0]; // message body starts from here @@ -182,7 +183,7 @@ typedef struct { #pragma pack(pop) typedef enum { Normal, Quit, Release, Register } STransMsgType; -typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken } ConnStatus; +typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } ConnStatus; #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) #define RPC_RESERVE_SIZE (sizeof(STranConnCtx)) @@ -277,38 +278,44 @@ void transCtxCleanup(STransCtx* ctx); void transCtxClear(STransCtx* ctx); void transCtxMerge(STransCtx* dst, STransCtx* src); void* transCtxDumpVal(STransCtx* ctx, int32_t key); +void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType); // queue sending msgs typedef struct { SArray* q; - void (*free)(void* arg); + void (*freeFunc)(const void* arg); } STransQueue; /* * init queue * note: queue'size is small, default 1 */ -void transQueueInit(STransQueue* queue, void (*free)(void* arg)); +void transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)); /* * put arg into queue * if queue'size > 1, return false; else return true */ bool transQueuePush(STransQueue* queue, void* arg); +/* + * the size of queue + */ +int32_t transQueueSize(STransQueue* queue); /* * pop head from queue */ - void* transQueuePop(STransQueue* queue); /* - * get head from queue + * get ith from queue */ -void* transQueueGet(STransQueue* queue); - +void* transQueueGet(STransQueue* queue, int i); +/* + * rm ith from queue + */ +void* transQueueRm(STransQueue* queue, int i); /* * queue empty or not */ - bool transQueueEmpty(STransQueue* queue); /* * clear queue diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index d629f6dba4..e02de2d8b9 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -60,21 +60,21 @@ void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, connHashMemPool = taosMemPoolInit(maxSessions, sizeof(SConnHash)); if (connHashMemPool == 0) return NULL; - connHashList = calloc(sizeof(SConnHash *), maxSessions); + connHashList = taosMemoryCalloc(sizeof(SConnHash *), maxSessions); if (connHashList == 0) { taosMemPoolCleanUp(connHashMemPool); return NULL; } - pCache = malloc(sizeof(SConnCache)); + pCache = taosMemoryMalloc(sizeof(SConnCache)); if (pCache == NULL) { taosMemPoolCleanUp(connHashMemPool); - free(connHashList); + taosMemoryFree(connHashList); return NULL; } memset(pCache, 0, sizeof(SConnCache)); - pCache->count = calloc(sizeof(int), maxSessions); + pCache->count = taosMemoryCalloc(sizeof(int), maxSessions); pCache->total = 0; pCache->keepTimer = keepTimer; pCache->maxSessions = maxSessions; @@ -82,7 +82,7 @@ void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, pCache->connHashList = connHashList; pCache->cleanFp = cleanFp; pCache->tmrCtrl = tmrCtrl; - pCache->lockedBy = calloc(sizeof(int64_t), maxSessions); + pCache->lockedBy = taosMemoryCalloc(sizeof(int64_t), maxSessions); taosTmrReset(rpcCleanConnCache, (int32_t)(pCache->keepTimer * 2), pCache, pCache->tmrCtrl, &pCache->pTimer); taosThreadMutexInit(&pCache->mutex, NULL); @@ -102,16 +102,16 @@ void rpcCloseConnCache(void *handle) { if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool); - tfree(pCache->connHashList); - tfree(pCache->count); - tfree(pCache->lockedBy); + taosMemoryFreeClear(pCache->connHashList); + taosMemoryFreeClear(pCache->count); + taosMemoryFreeClear(pCache->lockedBy); taosThreadMutexUnlock(&pCache->mutex); taosThreadMutexDestroy(&pCache->mutex); memset(pCache, 0, sizeof(SConnCache)); - free(pCache); + taosMemoryFree(pCache); } void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType) { diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 98b24ac3c7..6beda5bfed 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -208,7 +208,7 @@ static void rpcDecRef(SRpcInfo *pRpc); static void rpcFree(void *p) { tTrace("free mem: %p", p); - free(p); + taosMemoryFree(p); } static void rpcInitImp(void) { @@ -240,7 +240,7 @@ void *rpcOpen(const SRpcInit *pInit) { // taosThreadOnce(&tsRpcInit, rpcInit); - pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); + pRpc = (SRpcInfo *)taosMemoryCalloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; if (pInit->label) tstrncpy(pRpc->label, pInit->label, tListLen(pInit->label)); @@ -270,7 +270,7 @@ void *rpcOpen(const SRpcInit *pInit) { atomic_add_fetch_32(&tsRpcNum, 1); size_t size = sizeof(SRpcConn) * pRpc->sessions; - pRpc->connList = (SRpcConn *)calloc(1, size); + pRpc->connList = (SRpcConn *)taosMemoryCalloc(1, size); if (pRpc->connList == NULL) { tError("%s failed to allocate memory for taos connections, size:%" PRId64, pRpc->label, (int64_t)size); rpcClose(pRpc); @@ -350,7 +350,7 @@ void rpcClose(void *param) { void *rpcMallocCont(int contLen) { int size = contLen + RPC_MSG_OVERHEAD; - char *start = (char *)calloc(1, (size_t)size); + char *start = (char *)taosMemoryCalloc(1, (size_t)size); if (start == NULL) { tError("failed to malloc msg, size:%d", size); return NULL; @@ -364,7 +364,7 @@ void *rpcMallocCont(int contLen) { void rpcFreeCont(void *cont) { if (cont) { char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext); - free(temp); + taosMemoryFree(temp); tTrace("free mem: %p", temp); } } @@ -374,12 +374,12 @@ void *rpcReallocCont(void *ptr, int contLen) { char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead); if (contLen == 0) { - free(start); + taosMemoryFree(start); return NULL; } int size = contLen + RPC_MSG_OVERHEAD; - start = realloc(start, size); + start = taosMemoryRealloc(start, size); if (start == NULL) { tError("failed to realloc cont, size:%d", size); return NULL; @@ -574,7 +574,7 @@ void rpcCancelRequest(int64_t rid) { static void rpcFreeMsg(void *msg) { if (msg) { char *temp = (char *)msg - sizeof(SRpcReqContext); - free(temp); + taosMemoryFree(temp); tTrace("free mem: %p", temp); } } @@ -1039,7 +1039,7 @@ static void doRpcReportBrokenLinkToServer(void *param, void *id) { SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); SRpcInfo *pRpc = pConn->pRpc; (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL); - free(pRpcMsg); + taosMemoryFree(pRpcMsg); } static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; @@ -1049,7 +1049,7 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { rpcAddRef(pRpc); tDebug("%s, notify the server app, connection is gone", pConn->info); - SRpcMsg *rpcMsg = malloc(sizeof(SRpcMsg)); + SRpcMsg *rpcMsg = taosMemoryMalloc(sizeof(SRpcMsg)); rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length rpcMsg->ahandle = pConn->ahandle; @@ -1061,7 +1061,7 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { if (pRpc->cfp) { taosTmrStart(doRpcReportBrokenLinkToServer, 0, rpcMsg, pRpc->tmrCtrl); } else { - free(rpcMsg); + taosMemoryFree(rpcMsg); } } @@ -1484,7 +1484,7 @@ static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { return contLen; } - char *buf = malloc(contLen + overhead + 8); // 8 extra bytes + char *buf = taosMemoryMalloc(contLen + overhead + 8); // 8 extra bytes if (buf == NULL) { tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); return contLen; @@ -1510,7 +1510,7 @@ static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { finalLen = contLen; } - free(buf); + taosMemoryFree(buf); return finalLen; } @@ -1526,7 +1526,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { int contLen = htonl(pComp->contLen); // prepare the temporary buffer to decompress message - char *temp = (char *)malloc(contLen + RPC_MSG_OVERHEAD); + char *temp = (char *)taosMemoryMalloc(contLen + RPC_MSG_OVERHEAD); pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext if (pNewHead) { @@ -1671,10 +1671,10 @@ static void rpcDecRef(SRpcInfo *pRpc) { taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); - tfree(pRpc->connList); + taosMemoryFreeClear(pRpc->connList); taosThreadMutexDestroy(&pRpc->mutex); tDebug("%s rpc resources are released", pRpc->label); - tfree(pRpc); + taosMemoryFreeClear(pRpc); atomic_sub_fetch_32(&tsRpcNum, 1); } diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 4b1700a10f..52c5ddcf63 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -78,7 +78,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread SServerObj *pServerObj; SThreadObj *pThreadObj; - pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); + pServerObj = (SServerObj *)taosMemoryCalloc(sizeof(SServerObj), 1); if (pServerObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); @@ -92,11 +92,11 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread tstrncpy(pServerObj->label, label, sizeof(pServerObj->label)); pServerObj->numOfThreads = numOfThreads; - pServerObj->pThreadObj = (SThreadObj **)calloc(sizeof(SThreadObj *), numOfThreads); + pServerObj->pThreadObj = (SThreadObj **)taosMemoryCalloc(sizeof(SThreadObj *), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - free(pServerObj); + taosMemoryFree(pServerObj); return NULL; } @@ -107,13 +107,13 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread // initialize parameters in case it may encounter error later for (int i = 0; i < numOfThreads; ++i) { - pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), 1); + pThreadObj = (SThreadObj *)taosMemoryCalloc(sizeof(SThreadObj), 1); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j = 0; j < i; ++j) free(pServerObj->pThreadObj[j]); - free(pServerObj->pThreadObj); - free(pServerObj); + for (int j = 0; j < i; ++j) taosMemoryFree(pServerObj->pThreadObj[j]); + taosMemoryFree(pServerObj->pThreadObj); + taosMemoryFree(pServerObj); return NULL; } @@ -222,8 +222,8 @@ void taosCleanUpTcpServer(void *handle) { tDebug("%s TCP server is cleaned up", pServerObj->label); - tfree(pServerObj->pThreadObj); - tfree(pServerObj); + taosMemoryFreeClear(pServerObj->pThreadObj); + taosMemoryFreeClear(pServerObj); } static void *taosAcceptTcpConnection(void *arg) { @@ -290,7 +290,7 @@ static void *taosAcceptTcpConnection(void *arg) { } void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - SClientObj *pClientObj = (SClientObj *)calloc(1, sizeof(SClientObj)); + SClientObj *pClientObj = (SClientObj *)taosMemoryCalloc(1, sizeof(SClientObj)); if (pClientObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); @@ -299,10 +299,10 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); pClientObj->numOfThreads = numOfThreads; - pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj *)); + pClientObj->pThreadObj = (SThreadObj **)taosMemoryCalloc(numOfThreads, sizeof(SThreadObj *)); if (pClientObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - tfree(pClientObj); + taosMemoryFreeClear(pClientObj); terrno = TAOS_SYSTEM_ERROR(errno); } @@ -312,12 +312,12 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); for (int i = 0; i < numOfThreads; ++i) { - SThreadObj *pThreadObj = (SThreadObj *)calloc(1, sizeof(SThreadObj)); + SThreadObj *pThreadObj = (SThreadObj *)taosMemoryCalloc(1, sizeof(SThreadObj)); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j = 0; j < i; ++j) free(pClientObj->pThreadObj[j]); - free(pClientObj); + for (int j = 0; j < i; ++j) taosMemoryFree(pClientObj->pThreadObj[j]); + taosMemoryFree(pClientObj); taosThreadAttrDestroy(&thattr); return NULL; } @@ -378,8 +378,8 @@ void taosCleanUpTcpClient(void *chandle) { } tDebug("%s TCP client is cleaned up", pClientObj->label); - tfree(pClientObj->pThreadObj); - tfree(pClientObj); + taosMemoryFreeClear(pClientObj->pThreadObj); + taosMemoryFreeClear(pClientObj); } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { @@ -477,9 +477,9 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); int32_t size = msgLen + tsRpcOverhead; - buffer = malloc(size); + buffer = taosMemoryMalloc(size); if (NULL == buffer) { - tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); + tError("%s %p TCP taosMemoryMalloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { tTrace("%s %p read data, FD:%p TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, buffer); @@ -491,7 +491,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { if (leftLen != retLen) { tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); - free(buffer); + taosMemoryFree(buffer); return -1; } @@ -507,7 +507,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { pInfo->connType = RPC_CONN_TCP; if (pFdObj->closedByApp) { - free(buffer); + taosMemoryFree(buffer); return -1; } @@ -580,7 +580,7 @@ static void *taosProcessTcpData(void *param) { taosThreadMutexDestroy(&(pThreadObj->mutex)); tDebug("%s TCP thread exits ...", pThreadObj->label); - tfree(pThreadObj); + taosMemoryFreeClear(pThreadObj); return NULL; } @@ -588,7 +588,7 @@ static void *taosProcessTcpData(void *param) { static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { struct epoll_event event; - SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1); + SFdObj *pFdObj = (SFdObj *)taosMemoryCalloc(sizeof(SFdObj), 1); if (pFdObj == NULL) { return NULL; } @@ -601,7 +601,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; if (taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_ADD, pSocket, &event) < 0) { - tfree(pFdObj); + taosMemoryFreeClear(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -652,6 +652,6 @@ static void taosFreeFdObj(SFdObj *pFdObj) { tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); - tfree(pFdObj); + taosMemoryFreeClear(pFdObj); } #endif \ No newline at end of file diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index af26761603..359a94011d 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -62,7 +62,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads SUdpConnSet *pSet; int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); - pSet = (SUdpConnSet *)malloc((size_t)size); + pSet = (SUdpConnSet *)taosMemoryMalloc((size_t)size); if (pSet == NULL) { tError("%s failed to allocate UdpConn", label); terrno = TAOS_SYSTEM_ERROR(errno); @@ -92,7 +92,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads break; } - pConn->buffer = malloc(RPC_MAX_UDP_SIZE); + pConn->buffer = taosMemoryMalloc(RPC_MAX_UDP_SIZE); if (NULL == pConn->buffer) { tError("%s failed to malloc recv buffer", label); break; @@ -148,7 +148,7 @@ void taosStopUdpConnection(void *handle) { if (taosCheckPthreadValid(pConn->thread)) { taosThreadJoin(pConn->thread, NULL); } - tfree(pConn->buffer); + taosMemoryFreeClear(pConn->buffer); // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); } @@ -167,7 +167,7 @@ void taosCleanUpUdpConnection(void *handle) { } tDebug("%s UDP is cleaned up", pSet->label); - tfree(pSet); + taosMemoryFreeClear(pSet); } void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { @@ -219,7 +219,7 @@ static void *taosRecvUdpData(void *param) { } int32_t size = dataLen + tsRpcOverhead; - char * tmsg = malloc(size); + char * tmsg = taosMemoryMalloc(size); if (NULL == tmsg) { tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); continue; diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 4e2cb83527..90f15dd7d0 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -47,7 +47,7 @@ static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pH int32_t taosCompressHttpRport(char* pSrc, int32_t srcLen) { int32_t code = -1; int32_t destLen = srcLen; - void* pDest = malloc(destLen); + void* pDest = taosMemoryMalloc(destLen); if (pDest == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -109,7 +109,7 @@ _OVER: code = gzipStream.total_out; } - free(pDest); + taosMemoryFree(pDest); return code; } @@ -145,7 +145,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 uv_tcp_t socket_tcp = {0}; uv_loop_t* loop = uv_default_loop(); uv_tcp_init(loop, &socket_tcp); - uv_connect_t* connect = (uv_connect_t*)malloc(sizeof(uv_connect_t)); + uv_connect_t* connect = (uv_connect_t*)taosMemoryMalloc(sizeof(uv_connect_t)); if (flag == HTTP_GZIP) { int32_t dstLen = taosCompressHttpRport(pCont, contLen); @@ -168,7 +168,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 terrno = 0; uv_run(loop, UV_RUN_DEFAULT); uv_loop_close(loop); - free(connect); + taosMemoryFree(connect); return terrno; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 9d0fba4885..ebb90338cd 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -28,7 +28,7 @@ void (*taosUnRefHandle[])(void* handle) = {transUnrefSrvHandle, transUnrefCliHan void (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; void* rpcOpen(const SRpcInit* pInit) { - SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + SRpcInfo* pRpc = taosMemoryCalloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) { return NULL; } @@ -61,13 +61,13 @@ void* rpcOpen(const SRpcInit* pInit) { void rpcClose(void* arg) { SRpcInfo* pRpc = (SRpcInfo*)arg; (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); - free(pRpc); + taosMemoryFree(pRpc); return; } void* rpcMallocCont(int contLen) { int size = contLen + TRANS_MSG_OVERHEAD; - char* start = (char*)calloc(1, (size_t)size); + char* start = (char*)taosMemoryCalloc(1, (size_t)size); if (start == NULL) { tError("failed to malloc msg, size:%d", size); return NULL; @@ -81,7 +81,7 @@ void rpcFreeCont(void* cont) { if (cont == NULL) { return; } - free((char*)cont - TRANS_MSG_OVERHEAD); + taosMemoryFree((char*)cont - TRANS_MSG_OVERHEAD); } void* rpcReallocCont(void* ptr, int contLen) { if (ptr == NULL) { @@ -89,7 +89,7 @@ void* rpcReallocCont(void* ptr, int contLen) { } char* st = (char*)ptr - TRANS_MSG_OVERHEAD; int sz = contLen + TRANS_MSG_OVERHEAD; - st = realloc(st, sz); + st = taosMemoryRealloc(st, sz); if (st == NULL) { return NULL; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c0ee9b9ca5..d9c288a39b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -25,12 +25,11 @@ typedef struct SCliConn { void* hostThrd; SConnBuffer readBuf; void* data; - // SArray* cliMsgs; - STransQueue cliMsgs; - queue conn; - uint64_t expireTime; - int hThrdIdx; - STransCtx ctx; + STransQueue cliMsgs; + queue conn; + uint64_t expireTime; + int hThrdIdx; + STransCtx ctx; bool broken; // link broken or not ConnStatus status; // @@ -54,6 +53,7 @@ typedef struct SCliMsg { queue q; uint64_t st; STransMsgType type; + int sent; //(0: no send, 1: alread sent) } SCliMsg; typedef struct SCliThrdObj { @@ -136,6 +136,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd); #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + uint64_t ahandle = head->ahandle; \ + CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ conn->status = ConnRelease; \ transClearBuffer(&conn->readBuf); \ transFreeMsg(transContFromHead((char*)head)); \ @@ -147,10 +149,40 @@ static void destroyThrdObj(SCliThrdObj* pThrd); SCliThrdObj* thrd = conn->hostThrd; \ addConnToPool(thrd->pool, conn); \ } \ + destroyCmsg(pMsg); \ return; \ } \ } while (0) +#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \ + do { \ + int i = 0, sz = transQueueSize(&conn->cliMsgs); \ + for (; i < sz; i++) { \ + pMsg = transQueueGet(&conn->cliMsgs, i); \ + if (pMsg != NULL && pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \ + break; \ + } \ + } \ + if (i == sz) { \ + pMsg = NULL; \ + } else { \ + pMsg = transQueueRm(&conn->cliMsgs, i); \ + } \ + } while (0) +#define CONN_GET_NEXT_SENDMSG(conn) \ + do { \ + int i = 0; \ + do { \ + pCliMsg = transQueueGet(&conn->cliMsgs, i++); \ + if (pCliMsg && 0 == pCliMsg->sent) { \ + break; \ + } \ + } while (pCliMsg != NULL); \ + if (pCliMsg == NULL) { \ + goto _RETURN; \ + } \ + } while (0) + #define CONN_HANDLE_THREAD_QUIT(thrd) \ do { \ if (thrd->quit) { \ @@ -173,8 +205,10 @@ static void destroyThrdObj(SCliThrdObj* pThrd); transRefCliHandle(conn); \ } \ } while (0) -#define CONN_NO_PERSIST_BY_APP(conn) ((conn)->status == ConnNormal && T_REF_VAL_GET(conn) == 1) - +#define CONN_NO_PERSIST_BY_APP(conn) \ + (((conn)->status == ConnNormal || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) +#define CONN_RELEASE_BY_SERVER(conn) \ + (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) #define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) #define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) @@ -183,10 +217,13 @@ static void* cliWorkThread(void* arg); bool cliMaySendCachedMsg(SCliConn* conn) { if (!transQueueEmpty(&conn->cliMsgs)) { + SCliMsg* pCliMsg = NULL; + CONN_GET_NEXT_SENDMSG(conn); cliSend(conn); - return true; } return false; +_RETURN: + return false; } void cliHandleResp(SCliConn* conn) { SCliThrdObj* pThrd = conn->hostThrd; @@ -203,15 +240,38 @@ void cliHandleResp(SCliConn* conn) { transMsg.msgType = pHead->msgType; transMsg.ahandle = NULL; + SCliMsg* pMsg = NULL; + STransConnCtx* pCtx = NULL; CONN_SHOULD_RELEASE(conn, pHead); - SCliMsg* pMsg = transQueuePop(&conn->cliMsgs); - - STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; - if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); + if (CONN_NO_PERSIST_BY_APP(conn)) { + pMsg = transQueuePop(&conn->cliMsgs); + pCtx = pMsg ? pMsg->ctx : NULL; + if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { + transMsg.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); + if (transMsg.ahandle == NULL) { + transMsg.ahandle = transCtxDumpBrokenlinkVal(&conn->ctx, (int32_t*)&(transMsg.msgType)); + } + tDebug("cli conn %p construct ahandle %p, persist: 0", conn, transMsg.ahandle); + } else { + transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + tDebug("cli conn %p get ahandle %p, persist: 0", conn, transMsg.ahandle); + } } else { - transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + uint64_t ahandle = (uint64_t)pHead->ahandle; + CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); + if (pMsg == NULL) { + transMsg.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); + tDebug("cli conn %p construct ahandle %p by %d, persist: 1", conn, transMsg.ahandle, transMsg.msgType); + if (!CONN_RELEASE_BY_SERVER(conn) && transMsg.ahandle == NULL) { + transMsg.ahandle = transCtxDumpBrokenlinkVal(&conn->ctx, (int32_t*)&(transMsg.msgType)); + tDebug("cli conn %p construct ahandle %p due brokenlink, persist: 1", conn, transMsg.ahandle); + } + } else { + pCtx = pMsg ? pMsg->ctx : NULL; + transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; + tDebug("cli conn %p get ahandle %p, persist: 1", conn, transMsg.ahandle); + } } // buf's mem alread translated to transMsg.pCont transClearBuffer(&conn->readBuf); @@ -232,6 +292,11 @@ void cliHandleResp(SCliConn* conn) { // transUnrefCliHandle(conn); return; } + if (CONN_RELEASE_BY_SERVER(conn) && transMsg.ahandle == NULL) { + tTrace("except, server continue send while cli ignore it"); + // transUnrefCliHandle(conn); + return; + } if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, conn); @@ -256,41 +321,54 @@ void cliHandleResp(SCliConn* conn) { if (!uv_is_active((uv_handle_t*)&pThrd->timer) && pTransInst->idleTime > 0) { // uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } -_RETURN: - return; } void cliHandleExcept(SCliConn* pConn) { if (transQueueEmpty(&pConn->cliMsgs)) { - if (pConn->broken == true || CONN_NO_PERSIST_BY_APP(pConn)) { + if (pConn->broken == true && CONN_NO_PERSIST_BY_APP(pConn)) { + tTrace("%s cli conn %p handle except, persist:0", CONN_GET_INST_LABEL(pConn), pConn); transUnrefCliHandle(pConn); return; } } SCliThrdObj* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; - + bool once = false; do { SCliMsg* pMsg = transQueuePop(&pConn->cliMsgs); - + if (pMsg == NULL && once) { + break; + } STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; STransMsg transMsg = {0}; transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; transMsg.ahandle = NULL; + transMsg.handle = pConn; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { transMsg.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); + tDebug("%s cli conn %p construct ahandle %p by %s", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle, + TMSG_INFO(transMsg.msgType)); + if (transMsg.ahandle == NULL) { + transMsg.ahandle = transCtxDumpBrokenlinkVal(&pConn->ctx, (int32_t*)&(transMsg.msgType)); + tDebug("%s cli conn %p construct ahandle %p due to brokenlink", CONN_GET_INST_LABEL(pConn), pConn, + transMsg.ahandle); + } } else { transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; } if (pCtx == NULL || pCtx->pSem == NULL) { - tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); + tTrace("%s cli conn %p handle except", pTransInst->label, pConn); + if (transMsg.ahandle == NULL) { + once = true; + continue; + } (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { - tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, pConn); + tTrace("%s cli conn(sync) %p handle except", pTransInst->label, pConn); memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg)); tsem_post(pCtx->pSem); } @@ -364,8 +442,8 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { } queue* h = QUEUE_HEAD(&plist->conn); QUEUE_REMOVE(h); - SCliConn* conn = QUEUE_DATA(h, SCliConn, conn); + conn->status = ConnNormal; QUEUE_INIT(&conn->conn); return conn; } @@ -377,7 +455,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); transCtxCleanup(&conn->ctx); transQueueClear(&conn->cliMsgs); - conn->status = ConnNormal; + conn->status = ConnInPool; char key[128] = {0}; tstrncpy(key, conn->ip, strlen(conn->ip)); @@ -429,9 +507,9 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { } static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { - SCliConn* conn = calloc(1, sizeof(SCliConn)); + SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn)); // read/write stream handle - conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); + conn->stream = (uv_stream_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->stream->data = conn; @@ -456,17 +534,17 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { } static void cliDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; - free(conn->ip); - free(conn->stream); + taosMemoryFree(conn->ip); + taosMemoryFree(conn->stream); transCtxCleanup(&conn->ctx); transQueueDestroy(&conn->cliMsgs); tTrace("%s cli conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); - free(conn); + taosMemoryFree(conn); } static bool cliHandleNoResp(SCliConn* conn) { bool res = false; if (!transQueueEmpty(&conn->cliMsgs)) { - SCliMsg* pMsg = transQueueGet(&conn->cliMsgs); + SCliMsg* pMsg = transQueueGet(&conn->cliMsgs, 0); if (REQUEST_NO_RESP(&pMsg->msg)) { transQueuePop(&conn->cliMsgs); // taosArrayRemove(msgs, 0); @@ -504,7 +582,11 @@ void cliSend(SCliConn* pConn) { // assert(taosArrayGetSize(pConn->cliMsgs) > 0); assert(!transQueueEmpty(&pConn->cliMsgs)); - SCliMsg* pCliMsg = transQueueGet(&pConn->cliMsgs); + + SCliMsg* pCliMsg = NULL; + CONN_GET_NEXT_SENDMSG(pConn); + pCliMsg->sent = 1; + STransConnCtx* pCtx = pCliMsg->ctx; SCliThrdObj* pThrd = pConn->hostThrd; @@ -516,10 +598,12 @@ void cliSend(SCliConn* pConn) { pMsg->contLen = 0; } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - int msgLen = transMsgLenFromCont(pMsg->contLen); + pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; + + int msgLen = transMsgLenFromCont(pMsg->contLen); if (!pConn->secured) { - char* buf = calloc(1, msgLen + sizeof(STransUserMsg)); + char* buf = taosMemoryCalloc(1, msgLen + sizeof(STransUserMsg)); memcpy(buf, (char*)pHead, msgLen); STransUserMsg* uMsg = (STransUserMsg*)(buf + msgLen); @@ -555,6 +639,8 @@ void cliSend(SCliConn* pConn) { pConn->writeReq.data = pConn; uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); + return; +_RETURN: return; } @@ -643,6 +729,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { cliSend(conn); } else { conn = cliCreateConn(pThrd); + transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); conn->hThrdIdx = pCtx->hThrdIdx; @@ -697,12 +784,12 @@ static void* cliWorkThread(void* arg) { } void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - SCliObj* cli = calloc(1, sizeof(SCliObj)); + SCliObj* cli = taosMemoryCalloc(1, sizeof(SCliObj)); STrans* pTransInst = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; - cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); + cli->pThreadObj = (SCliThrdObj**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrdObj*)); for (int i = 0; i < cli->numOfThreads; i++) { SCliThrdObj* pThrd = createThrdObj(); @@ -731,16 +818,16 @@ static void destroyCmsg(SCliMsg* pMsg) { } transDestroyConnCtx(pMsg->ctx); destroyUserdata(&pMsg->msg); - free(pMsg); + taosMemoryFree(pMsg); } static SCliThrdObj* createThrdObj() { - SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); + SCliThrdObj* pThrd = (SCliThrdObj*)taosMemoryCalloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); taosThreadMutexInit(&pThrd->msgMtx, NULL); - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, cliAsyncCb); @@ -763,20 +850,20 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { transDestroyAsyncPool(pThrd->asyncPool); uv_timer_stop(&pThrd->timer); - free(pThrd->loop); - free(pThrd); + taosMemoryFree(pThrd->loop); + taosMemoryFree(pThrd); } static void transDestroyConnCtx(STransConnCtx* ctx) { if (ctx != NULL) { - free(ctx->ip); + taosMemoryFree(ctx->ip); } - free(ctx); + taosMemoryFree(ctx); } // void cliSendQuit(SCliThrdObj* thrd) { // cli can stop gracefully - SCliMsg* msg = calloc(1, sizeof(SCliMsg)); + SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); msg->type = Quit; transSendAsync(thrd->asyncPool, &msg->q); } @@ -795,8 +882,8 @@ void transCloseClient(void* arg) { cliSendQuit(cli->pThreadObj[i]); destroyThrdObj(cli->pThreadObj[i]); } - free(cli->pThreadObj); - free(cli); + taosMemoryFree(cli->pThreadObj); + taosMemoryFree(cli); } void transRefCliHandle(void* handle) { if (handle == NULL) { @@ -822,7 +909,7 @@ void transReleaseCliHandle(void* handle) { } STransMsg tmsg = {.handle = handle}; - SCliMsg* cmsg = calloc(1, sizeof(SCliMsg)); + SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; cmsg->type = Release; @@ -836,7 +923,7 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p index = cliRBChoseIdx(pTransInst); } - STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); + STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; pCtx->ip = strdup(ip); @@ -848,7 +935,7 @@ void transSendRequest(void* shandle, const char* ip, uint32_t port, STransMsg* p } assert(pTransInst->connType == TAOS_CONN_CLIENT); - SCliMsg* cliMsg = calloc(1, sizeof(SCliMsg)); + SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pMsg; cliMsg->st = taosGetTimestampUs(); @@ -867,17 +954,17 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq index = cliRBChoseIdx(pTransInst); } - STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); + STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->ahandle = pReq->ahandle; pCtx->msgType = pReq->msgType; pCtx->ip = strdup(ip); pCtx->port = port; pCtx->hThrdIdx = index; - pCtx->pSem = calloc(1, sizeof(tsem_t)); + pCtx->pSem = taosMemoryCalloc(1, sizeof(tsem_t)); pCtx->pRsp = pRsp; tsem_init(pCtx->pSem, 0, 0); - SCliMsg* cliMsg = calloc(1, sizeof(SCliMsg)); + SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pReq; cliMsg->st = taosGetTimestampUs(); @@ -888,7 +975,7 @@ void transSendRecv(void* shandle, const char* ip, uint32_t port, STransMsg* pReq tsem_t* pSem = pCtx->pSem; tsem_wait(pSem); tsem_destroy(pSem); - free(pSem); + taosMemoryFree(pSem); } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 209475ca05..3078324c6b 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -52,7 +52,7 @@ bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { return succ; } - char* buf = malloc(len + overhead + 8); // 8 extra bytes + char* buf = taosMemoryMalloc(len + overhead + 8); // 8 extra bytes if (buf == NULL) { tError("failed to allocate memory for rpc msg compression, contLen:%d", len); *flen = len; @@ -78,7 +78,7 @@ bool transCompressMsg(char* msg, int32_t len, int32_t* flen) { *flen = len; succ = false; } - free(buf); + taosMemoryFree(buf); return succ; } bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { @@ -92,15 +92,15 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen) { } void transConnCtxDestroy(STransConnCtx* ctx) { - free(ctx->ip); - free(ctx); + taosMemoryFree(ctx->ip); + taosMemoryFree(ctx); } void transFreeMsg(void* msg) { if (msg == NULL) { return; } - free((char*)msg - sizeof(STransMsgHead)); + taosMemoryFree((char*)msg - sizeof(STransMsgHead)); } int transInitBuffer(SConnBuffer* buf) { @@ -123,7 +123,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { SConnBuffer* p = connBuf; if (p->cap == 0) { - p->buf = (char*)calloc(CAPACITY, sizeof(char)); + p->buf = (char*)taosMemoryCalloc(CAPACITY, sizeof(char)); p->len = 0; p->cap = CAPACITY; p->total = -1; @@ -135,7 +135,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { uvBuf->len = CAPACITY - p->len; } else { p->cap = p->total; - p->buf = realloc(p->buf, p->cap); + p->buf = taosMemoryRealloc(p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; @@ -160,7 +160,7 @@ int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) { return 0; } int transUnpackMsg(STransMsgHead* msgHead) { return 0; } int transDestroyBuffer(SConnBuffer* buf) { if (buf->cap > 0) { - tfree(buf->buf); + taosMemoryFreeClear(buf->buf); } transClearBuffer(buf); @@ -174,16 +174,16 @@ int transSetConnOption(uv_tcp_t* stream) { } SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) { - SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); + SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); pool->index = 0; pool->nAsync = sz; - pool->asyncs = calloc(1, sizeof(uv_async_t) * pool->nAsync); + pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); for (int i = 0; i < pool->nAsync; i++) { uv_async_t* async = &(pool->asyncs[i]); uv_async_init(loop, async, cb); - SAsyncItem* item = calloc(1, sizeof(SAsyncItem)); + SAsyncItem* item = taosMemoryCalloc(1, sizeof(SAsyncItem)); item->pThrd = arg; QUEUE_INIT(&item->qmsg); taosThreadMutexInit(&item->mtx, NULL); @@ -198,10 +198,10 @@ void transDestroyAsyncPool(SAsyncPool* pool) { SAsyncItem* item = async->data; taosThreadMutexDestroy(&item->mtx); - free(item); + taosMemoryFree(item); } - free(pool->asyncs); - free(pool); + taosMemoryFree(pool->asyncs); + taosMemoryFree(pool); } int transSendAsync(SAsyncPool* pool, queue* q) { int idx = pool->index; @@ -235,15 +235,18 @@ void transCtxCleanup(STransCtx* ctx) { STransCtxVal* iter = taosHashIterate(ctx->args, NULL); while (iter) { - iter->free(iter->val); + iter->freeFunc(iter->val); iter = taosHashIterate(ctx->args, iter); } + taosHashCleanup(ctx->args); + ctx->args = NULL; } void transCtxMerge(STransCtx* dst, STransCtx* src) { if (dst->args == NULL) { dst->args = src->args; + dst->brokenVal = src->brokenVal; src->args = NULL; return; } @@ -256,7 +259,7 @@ void transCtxMerge(STransCtx* dst, STransCtx* src) { STransCtxVal* dVal = taosHashGet(dst->args, key, klen); if (dVal) { - dVal->free(dVal->val); + dVal->freeFunc(dVal->val); } taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); iter = taosHashIterate(src->args, iter); @@ -271,16 +274,30 @@ void* transCtxDumpVal(STransCtx* ctx, int32_t key) { if (cVal == NULL) { return NULL; } - char* ret = calloc(1, cVal->len); - memcpy(ret, (char*)cVal->val, cVal->len); - return (void*)ret; + void* ret = NULL; + (*cVal->clone)(cVal->val, &ret); + return ret; +} +void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { + void* ret = NULL; + if (ctx->brokenVal.clone == NULL) { + return ret; + } + (*ctx->brokenVal.clone)(ctx->brokenVal.val, &ret); + + *msgType = ctx->brokenVal.msgType; + + return ret; } -void transQueueInit(STransQueue* queue, void (*free)(void* arg)) { +void transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) { queue->q = taosArrayInit(2, sizeof(void*)); - queue->free = free; + queue->freeFunc = freeFunc; } bool transQueuePush(STransQueue* queue, void* arg) { + if (queue->q == NULL) { + return true; + } taosArrayPush(queue->q, &arg); if (taosArrayGetSize(queue->q) > 1) { return false; @@ -288,30 +305,54 @@ bool transQueuePush(STransQueue* queue, void* arg) { return true; } void* transQueuePop(STransQueue* queue) { - if (taosArrayGetSize(queue->q) == 0) { + if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { return NULL; } void* ptr = taosArrayGetP(queue->q, 0); taosArrayRemove(queue->q, 0); return ptr; } - -void* transQueueGet(STransQueue* queue) { - if (taosArrayGetSize(queue->q) == 0) { +int32_t transQueueSize(STransQueue* queue) { + if (queue->q == NULL) { + return 0; + } + return taosArrayGetSize(queue->q); +} +void* transQueueGet(STransQueue* queue, int i) { + if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { return NULL; } - void* ptr = taosArrayGetP(queue->q, 0); + if (i >= taosArrayGetSize(queue->q)) { + return NULL; + } + + void* ptr = taosArrayGetP(queue->q, i); return ptr; } + +void* transQueueRm(STransQueue* queue, int i) { + if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + return NULL; + } + if (i >= taosArrayGetSize(queue->q)) { + return NULL; + } + void* ptr = taosArrayGetP(queue->q, i); + taosArrayRemove(queue->q, i); + return ptr; +} + bool transQueueEmpty(STransQueue* queue) { - // + if (queue->q == NULL) { + return true; + } return taosArrayGetSize(queue->q) == 0; } void transQueueClear(STransQueue* queue) { - if (queue->free != NULL) { + if (queue->freeFunc != NULL) { for (int i = 0; i < taosArrayGetSize(queue->q); i++) { void* p = taosArrayGetP(queue->q, i); - queue->free(p); + queue->freeFunc(p); } } taosArrayClear(queue->q); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index c6032a9569..c137657a99 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -93,25 +93,25 @@ typedef struct SServerObj { static const char* notify = "a"; -#define CONN_SHOULD_RELEASE(conn, head) \ - do { \ - if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - conn->status = ConnRelease; \ - transClearBuffer(&conn->readBuf); \ - transFreeMsg(transContFromHead((char*)head)); \ - tTrace("server conn %p received release request", conn); \ - \ - STransMsg tmsg = {.handle = (void*)conn, .code = 0}; \ - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); \ - srvMsg->msg = tmsg; \ - srvMsg->type = Release; \ - srvMsg->pConn = conn; \ - if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ - return; \ - } \ - uvStartSendRespInternal(srvMsg); \ - return; \ - } \ +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + tTrace("server conn %p received release request", conn); \ + \ + STransMsg tmsg = {.code = 0, .handle = (void*)conn, .ahandle = NULL}; \ + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); \ + srvMsg->msg = tmsg; \ + srvMsg->type = Release; \ + srvMsg->pConn = conn; \ + if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ + return; \ + } \ + uvStartSendRespInternal(srvMsg); \ + return; \ + } \ } while (0) static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); @@ -190,7 +190,7 @@ static void uvHandleReq(SSrvConn* pConn) { transMsg.pCont = pHead->content; transMsg.msgType = pHead->msgType; transMsg.code = pHead->code; - transMsg.ahandle = NULL; + transMsg.ahandle = (void*)pHead->ahandle; transMsg.handle = NULL; transClearBuffer(&pConn->readBuf); @@ -199,6 +199,7 @@ static void uvHandleReq(SSrvConn* pConn) { if (pHead->persist == 1) { pConn->status = ConnAcquire; transRefSrvHandle(pConn); + tDebug("server conn %p acquired by server app", pConn); } } if (pConn->status == ConnNormal && pHead->noResp == 0) { @@ -256,7 +257,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { buf->len = 2; - buf->base = calloc(1, sizeof(char) * buf->len); + buf->base = taosMemoryCalloc(1, sizeof(char) * buf->len); } void uvOnTimeoutCb(uv_timer_t* handle) { @@ -279,7 +280,7 @@ void uvOnSendCb(uv_write_t* req, int status) { destroySmsg(msg); // send second data, just use for push if (!transQueueEmpty(&conn->srvMsgs)) { - msg = (SSrvMsg*)transQueueGet(&conn->srvMsgs); + msg = (SSrvMsg*)transQueueGet(&conn->srvMsgs, 0); if (msg->type == Register && conn->status == ConnAcquire) { conn->regArg.notifyCount = 0; conn->regArg.init = 1; @@ -290,7 +291,12 @@ void uvOnSendCb(uv_write_t* req, int status) { memset(&conn->regArg, 0, sizeof(conn->regArg)); } transQueuePop(&conn->srvMsgs); - free(msg); + taosMemoryFree(msg); + + msg = (SSrvMsg*)transQueueGet(&conn->srvMsgs, 0); + if (msg != NULL) { + uvStartSendRespInternal(msg); + } } else { uvStartSendRespInternal(msg); } @@ -308,7 +314,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { } else { tError("fail to dispatch conn to work thread"); } - free(req); + taosMemoryFree(req); } static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { @@ -321,6 +327,7 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { pMsg->contLen = 0; } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + pHead->ahandle = (uint64_t)pMsg->ahandle; // pHead->secured = pMsg->code == 0 ? 1 : 0; // if (!pConn->secured) { @@ -380,7 +387,7 @@ static void destroySmsg(SSrvMsg* smsg) { return; } transFreeMsg(smsg->msg.pCont); - free(smsg); + taosMemoryFree(smsg); } static void destroyAllConn(SWorkThrdObj* pThrd) { while (!QUEUE_IS_EMPTY(&pThrd->conn)) { @@ -430,7 +437,7 @@ static void uvShutDownCb(uv_shutdown_t* req, int status) { tDebug("conn failed to shut down: %s", uv_err_name(status)); } uv_close((uv_handle_t*)req->handle, uvDestroyConn); - free(req); + taosMemoryFree(req); } void uvOnAcceptCb(uv_stream_t* stream, int status) { @@ -439,11 +446,11 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { } SServerObj* pObj = container_of(stream, SServerObj, server); - uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_t* cli = (uv_tcp_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pObj->loop, cli); if (uv_accept(stream, (uv_stream_t*)cli) == 0) { - uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); + uv_write_t* wr = (uv_write_t*)taosMemoryMalloc(sizeof(uv_write_t)); uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); @@ -453,7 +460,7 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); } else { uv_close((uv_handle_t*)cli, NULL); - free(cli); + taosMemoryFree(cli); } } void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { @@ -469,7 +476,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { // free memory allocated by assert(nread == strlen(notify)); assert(buf->base[0] == notify[0]); - free(buf->base); + taosMemoryFree(buf->base); SWorkThrdObj* pThrd = q->data; @@ -492,7 +499,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->hostThrd = pThrd; // init client handle - pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + pConn->pTcp = (uv_tcp_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, pConn->pTcp); pConn->pTcp->data = pConn; @@ -537,7 +544,7 @@ void* acceptThread(void* arg) { } static bool addHandleToWorkloop(void* arg) { SWorkThrdObj* pThrd = arg; - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); if (0 != uv_loop_init(pThrd->loop)) { return false; } @@ -553,7 +560,7 @@ static bool addHandleToWorkloop(void* arg) { // conn set QUEUE_INIT(&pThrd->conn); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; } @@ -569,7 +576,7 @@ static bool addHandleToAcceptloop(void* arg) { } // register an async here to quit server gracefully - srv->pAcceptAsync = calloc(1, sizeof(uv_async_t)); + srv->pAcceptAsync = taosMemoryCalloc(1, sizeof(uv_async_t)); uv_async_init(srv->loop, srv->pAcceptAsync, uvAcceptAsyncCb); srv->pAcceptAsync->data = srv; @@ -596,7 +603,7 @@ void* workerThread(void* arg) { static SSrvConn* createConn(void* hThrd) { SWorkThrdObj* pThrd = hThrd; - SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + SSrvConn* pConn = (SSrvConn*)taosMemoryCalloc(1, sizeof(SSrvConn)); QUEUE_INIT(&pConn->queue); QUEUE_PUSH(&pThrd->conn, &pConn->queue); @@ -617,11 +624,9 @@ static void destroyConn(SSrvConn* conn, bool clear) { return; } transDestroyBuffer(&conn->readBuf); - - transQueueDestroy(&conn->srvMsgs); if (clear) { tTrace("server conn %p to be destroyed", conn); - uv_shutdown_t* req = malloc(sizeof(uv_shutdown_t)); + uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } } @@ -634,33 +639,35 @@ static void uvDestroyConn(uv_handle_t* handle) { tDebug("server conn %p destroy", conn); uv_timer_stop(&conn->pTimer); + transQueueDestroy(&conn->srvMsgs); QUEUE_REMOVE(&conn->queue); - free(conn->pTcp); - // free(conn); + taosMemoryFree(conn->pTcp); + // taosMemoryFree(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { + tTrace("work thread quit"); uv_loop_close(thrd->loop); uv_stop(thrd->loop); } } void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { - SServerObj* srv = calloc(1, sizeof(SServerObj)); - srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + SServerObj* srv = taosMemoryCalloc(1, sizeof(SServerObj)); + srv->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); srv->numOfThreads = numOfThreads; srv->workerIdx = 0; - srv->pThreadObj = (SWorkThrdObj**)calloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); - srv->pipe = (uv_pipe_t**)calloc(srv->numOfThreads, sizeof(uv_pipe_t*)); + srv->pThreadObj = (SWorkThrdObj**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); + srv->pipe = (uv_pipe_t**)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t*)); srv->ip = ip; srv->port = port; uv_loop_init(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj)); + SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); thrd->quit = false; srv->pThreadObj[i] = thrd; - srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; @@ -700,14 +707,14 @@ End: return NULL; } void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { + thrd->quit = true; if (QUEUE_IS_EMPTY(&thrd->conn)) { uv_loop_close(thrd->loop); uv_stop(thrd->loop); } else { destroyAllConn(thrd); - thrd->quit = true; } - free(msg); + taosMemoryFree(msg); } void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) { // release handle to rpc init @@ -725,7 +732,7 @@ void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) { } void uvHandleResp(SSrvMsg* msg, SWorkThrdObj* thrd) { // send msg to client - tDebug("server conn %p start to send resp", msg->pConn); + tDebug("server conn %p start to send resp (2/2)", msg->pConn); uvStartSendResp(msg); } void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd) { @@ -735,16 +742,18 @@ void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd) { if (!transQueuePush(&conn->srvMsgs, msg)) { return; } + transQueuePop(&conn->srvMsgs); conn->regArg.notifyCount = 0; conn->regArg.init = 1; conn->regArg.msg = msg->msg; + tDebug("server conn %p register brokenlink callback succ", conn); if (conn->broken) { STrans* pTransInst = conn->pTransInst; (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); memset(&conn->regArg, 0, sizeof(conn->regArg)); } - free(msg); + taosMemoryFree(msg); } } void destroyWorkThrd(SWorkThrdObj* pThrd) { @@ -752,12 +761,12 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { return; } taosThreadJoin(pThrd->thread, NULL); - free(pThrd->loop); + taosMemoryFree(pThrd->loop); transDestroyAsyncPool(pThrd->asyncPool); - free(pThrd); + taosMemoryFree(pThrd); } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { - SSrvMsg* msg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* msg = taosMemoryCalloc(1, sizeof(SSrvMsg)); msg->type = Quit; tDebug("server send quit msg to work thread"); transSendAsync(pThrd->asyncPool, &msg->q); @@ -766,25 +775,26 @@ void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { void transCloseServer(void* arg) { // impl later SServerObj* srv = arg; - for (int i = 0; i < srv->numOfThreads; i++) { - sendQuitToWorkThrd(srv->pThreadObj[i]); - destroyWorkThrd(srv->pThreadObj[i]); - } tDebug("send quit msg to accept thread"); uv_async_send(srv->pAcceptAsync); taosThreadJoin(srv->thread, NULL); - free(srv->pThreadObj); - free(srv->pAcceptAsync); - free(srv->loop); + for (int i = 0; i < srv->numOfThreads; i++) { + sendQuitToWorkThrd(srv->pThreadObj[i]); + destroyWorkThrd(srv->pThreadObj[i]); + } + + taosMemoryFree(srv->pThreadObj); + taosMemoryFree(srv->pAcceptAsync); + taosMemoryFree(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { - free(srv->pipe[i]); + taosMemoryFree(srv->pipe[i]); } - free(srv->pipe); + taosMemoryFree(srv->pipe); - free(srv); + taosMemoryFree(srv); } void transRefSrvHandle(void* handle) { @@ -815,9 +825,9 @@ void transReleaseSrvHandle(void* handle) { SSrvConn* pConn = handle; SWorkThrdObj* pThrd = pConn->hostThrd; - STransMsg tmsg = {.handle = handle, .code = 0}; + STransMsg tmsg = {.code = 0, .handle = handle, .ahandle = NULL}; - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->msg = tmsg; srvMsg->type = Release; srvMsg->pConn = pConn; @@ -831,12 +841,15 @@ void transSendResponse(const STransMsg* pMsg) { } SSrvConn* pConn = pMsg->handle; SWorkThrdObj* pThrd = pConn->hostThrd; + if (pThrd->quit) { + return; + } - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; srvMsg->type = Normal; - tTrace("server conn %p start to send resp", pConn); + tTrace("server conn %p start to send resp (1/2)", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); } void transRegisterMsg(const STransMsg* msg) { @@ -846,7 +859,7 @@ void transRegisterMsg(const STransMsg* msg) { SSrvConn* pConn = msg->handle; SWorkThrdObj* pThrd = pConn->hostThrd; - SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *msg; srvMsg->type = Register; diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 4af316acae..7d3c3aa012 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -184,7 +184,7 @@ int main(int argc, char *argv[]) { taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); + SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo) * appThreads); taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index f3be4e3452..e7fc7b45e1 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -161,7 +161,7 @@ int main(int argc, char *argv[]) { taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec*1000000 + systemTime.tv_usec; - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo)*appThreads); + SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo)*appThreads); taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index 14f32df0b8..27a468a86f 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -184,7 +184,7 @@ int main(int argc, char *argv[]) { taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); + SInfo *pInfo = (SInfo *)taosMemoryCalloc(1, sizeof(SInfo) * appThreads); taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); diff --git a/source/libs/transport/test/transUT.cpp b/source/libs/transport/test/transUT.cpp index 1c59879f40..9dbebd6cfe 100644 --- a/source/libs/transport/test/transUT.cpp +++ b/source/libs/transport/test/transUT.cpp @@ -367,9 +367,10 @@ TEST_F(TransEnv, srvReleaseHandle) { SRpcMsg resp = {0}; tr->SetSrvContinueSend(processReleaseHandleCb); // tr->Restart(processReleaseHandleCb); - void *handle = NULL; + void * handle = NULL; + SRpcMsg req = {0}; for (int i = 0; i < 1; i++) { - SRpcMsg req = {0}; + memset(&req, 0, sizeof(req)); req.handle = resp.handle; req.persistHandle = 1; req.msgType = 1; @@ -383,8 +384,9 @@ TEST_F(TransEnv, srvReleaseHandle) { } TEST_F(TransEnv, cliReleaseHandleExcept) { SRpcMsg resp = {0}; + SRpcMsg req = {0}; for (int i = 0; i < 3; i++) { - SRpcMsg req = {0}; + memset(&req, 0, sizeof(req)); req.handle = resp.handle; req.persistHandle = 1; req.msgType = 1; @@ -403,8 +405,10 @@ TEST_F(TransEnv, cliReleaseHandleExcept) { } TEST_F(TransEnv, srvContinueSend) { tr->SetSrvContinueSend(processContinueSend); + SRpcMsg req = {0}, resp = {0}; for (int i = 0; i < 10; i++) { - SRpcMsg req = {0}, resp = {0}; + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); req.msgType = 1; req.pCont = rpcMallocCont(10); req.contLen = 10; @@ -417,8 +421,9 @@ TEST_F(TransEnv, srvPersistHandleExcept) { tr->SetSrvContinueSend(processContinueSend); // tr->SetCliPersistFp(cliPersistHandle); SRpcMsg resp = {0}; + SRpcMsg req = {0}; for (int i = 0; i < 5; i++) { - SRpcMsg req = {0}; + memset(&req, 0, sizeof(req)); req.handle = resp.handle; req.msgType = 1; req.pCont = rpcMallocCont(10); @@ -436,8 +441,9 @@ TEST_F(TransEnv, srvPersistHandleExcept) { TEST_F(TransEnv, cliPersistHandleExcept) { tr->SetSrvContinueSend(processContinueSend); SRpcMsg resp = {0}; + SRpcMsg req = {0}; for (int i = 0; i < 5; i++) { - SRpcMsg req = {0}; + memset(&req, 0, sizeof(req)); req.handle = resp.handle; req.msgType = 1; req.pCont = rpcMallocCont(10); @@ -459,8 +465,9 @@ TEST_F(TransEnv, multiCliPersistHandleExcept) { TEST_F(TransEnv, queryExcept) { tr->SetSrvContinueSend(processRegisterFailure); SRpcMsg resp = {0}; + SRpcMsg req = {0}; for (int i = 0; i < 5; i++) { - SRpcMsg req = {0}; + memset(&req, 0, sizeof(req)); req.handle = resp.handle; req.persistHandle = 1; req.msgType = 1; @@ -477,8 +484,9 @@ TEST_F(TransEnv, queryExcept) { } TEST_F(TransEnv, noResp) { SRpcMsg resp = {0}; + SRpcMsg req = {0}; for (int i = 0; i < 5; i++) { - SRpcMsg req = {0}; + memset(&req, 0, sizeof(req)); req.noResp = 1; req.msgType = 1; req.pCont = rpcMallocCont(10); diff --git a/source/libs/transport/test/transportTests.cpp b/source/libs/transport/test/transportTests.cpp index 65d9302994..4165e6d8c9 100644 --- a/source/libs/transport/test/transportTests.cpp +++ b/source/libs/transport/test/transportTests.cpp @@ -90,7 +90,7 @@ TEST_F(QueueEnv, testPushAndPop) { assert(q->IsEmpty()); for (int i = 0; i < 100; i++) { - QueueElem *el = (QueueElem *)malloc(sizeof(QueueElem)); + QueueElem *el = (QueueElem *)taosMemoryMalloc(sizeof(QueueElem)); el->val = i; q->Push(el); } @@ -98,7 +98,7 @@ TEST_F(QueueEnv, testPushAndPop) { while (!q->IsEmpty()) { QueueElem *el = q->Pop(); assert(el->val == i++); - free(el); + taosMemoryFree(el); } assert(q->IsEmpty()); } @@ -109,7 +109,7 @@ TEST_F(QueueEnv, testRm) { assert(q->IsEmpty()); for (int i = 0; i < 100; i++) { - QueueElem *el = (QueueElem *)malloc(sizeof(QueueElem)); + QueueElem *el = (QueueElem *)taosMemoryMalloc(sizeof(QueueElem)); el->val = i; q->Push(el); set.push_back(el); @@ -117,7 +117,7 @@ TEST_F(QueueEnv, testRm) { for (int i = set.size() - 1; i >= 0; i--) { QueueElem *el = set[i]; q->RmElem(el); - free(el); + taosMemoryFree(el); } assert(q->IsEmpty()); } @@ -126,7 +126,7 @@ TEST_F(QueueEnv, testIter) { assert(q->IsEmpty()); std::vector vals; for (int i = 0; i < 100; i++) { - QueueElem *el = (QueueElem *)malloc(sizeof(QueueElem)); + QueueElem *el = (QueueElem *)taosMemoryMalloc(sizeof(QueueElem)); el->val = i; q->Push(el); vals.push_back(i); @@ -139,7 +139,7 @@ TEST_F(QueueEnv, testIter) { class TransCtxEnv : public ::testing::Test { protected: virtual void SetUp() { - ctx = (STransCtx *)calloc(1, sizeof(STransCtx)); + ctx = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(ctx); // TODO } @@ -153,79 +153,73 @@ class TransCtxEnv : public ::testing::Test { TEST_F(TransCtxEnv, mergeTest) { int key = 1; { - STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = malloc(12); - val1.len = 12; + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = malloc(12); - val1.len = 12; + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } transCtxMerge(ctx, src); - free(src); + taosMemoryFree(src); } EXPECT_EQ(2, taosHashGetSize(ctx->args)); { - STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = malloc(12); - val1.len = 12; + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = malloc(12); - val1.len = 12; + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } transCtxMerge(ctx, src); - free(src); + taosMemoryFree(src); } std::string val("Hello"); EXPECT_EQ(4, taosHashGetSize(ctx->args)); { key = 1; - STransCtx *src = (STransCtx *)calloc(1, sizeof(STransCtx)); + STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = calloc(1, 11); + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); - val1.len = 11; taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .len = 0, .free = free}; - val1.val = calloc(1, 11); + STransCtxVal val1 = {.val = NULL, .freeFunc = taosMemoryFree}; + val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); - val1.len = 11; taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } transCtxMerge(ctx, src); - free(src); + taosMemoryFree(src); } EXPECT_EQ(4, taosHashGetSize(ctx->args)); char *skey = (char *)transCtxDumpVal(ctx, 1); EXPECT_EQ(0, strcmp(skey, val.c_str())); - free(skey); + taosMemoryFree(skey); skey = (char *)transCtxDumpVal(ctx, 2); EXPECT_EQ(0, strcmp(skey, val.c_str())); diff --git a/source/libs/transport/test/uv.c b/source/libs/transport/test/uv.c index 5a19b7998c..fb026ef1a6 100644 --- a/source/libs/transport/test/uv.c +++ b/source/libs/transport/test/uv.c @@ -38,7 +38,7 @@ void echo_write(uv_write_t *req, int status) { fprintf(stderr, "Write error %s\n", uv_err_name(status)); } printf("write data to client\n"); - free(req); + taosMemoryFree(req); } void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { @@ -47,14 +47,14 @@ void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { pConn->ref += 1; printf("read data %d\n", nread, buf->base, buf->len); if (nread > 0) { - uv_write_t *req = (uv_write_t *)malloc(sizeof(uv_write_t)); + uv_write_t *req = (uv_write_t *)taosMemoryMalloc(sizeof(uv_write_t)); // dispatch request to database other process thread // just write out uv_buf_t write_out; write_out.base = buf->base; write_out.len = nread; uv_write((uv_write_t *)req, client, &write_out, 1, echo_write); - free(buf->base); + taosMemoryFree(buf->base); return; } @@ -63,11 +63,11 @@ void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { fprintf(stderr, "Read error %s\n", uv_err_name(nread)); uv_close((uv_handle_t *)client, NULL); } - free(buf->base); + taosMemoryFree(buf->base); } void alloc_buffer(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { - buf->base = malloc(suggested_size); + buf->base = taosMemoryMalloc(suggested_size); buf->len = suggested_size; } @@ -79,10 +79,10 @@ void on_new_connection(uv_stream_t *s, int status) { SServerObj *pObj = container_of(s, SServerObj, server); printf("new_connection from client\n"); - uv_tcp_t *client = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + uv_tcp_t *client = (uv_tcp_t *)taosMemoryMalloc(sizeof(uv_tcp_t)); uv_tcp_init(pObj->loop, client); if (uv_accept(s, (uv_stream_t *)client) == 0) { - uv_write_t *write_req = (uv_write_t *)malloc(sizeof(uv_write_t)); + uv_write_t *write_req = (uv_write_t *)taosMemoryMalloc(sizeof(uv_write_t)); uv_buf_t dummy_buf = uv_buf_init("a", 1); // despatch to worker thread pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; @@ -112,13 +112,13 @@ void child_on_new_connection(uv_stream_t *q, ssize_t nread, uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SConnCtx *pConn = malloc(sizeof(SConnCtx)); + SConnCtx *pConn = taosMemoryMalloc(sizeof(SConnCtx)); /* init conn timer*/ - pConn->pTimer = malloc(sizeof(uv_timer_t)); + pConn->pTimer = taosMemoryMalloc(sizeof(uv_timer_t)); uv_timer_init(pObj->loop, pConn->pTimer); - pConn->pClient = (uv_tcp_t *)malloc(sizeof(uv_tcp_t)); + pConn->pClient = (uv_tcp_t *)taosMemoryMalloc(sizeof(uv_tcp_t)); pConn->pWorkerAsync = pObj->workerAsync; // thread safty uv_tcp_init(pObj->loop, pConn->pClient); @@ -130,10 +130,10 @@ void child_on_new_connection(uv_stream_t *q, ssize_t nread, uv_read_start((uv_stream_t *)(pConn->pClient), alloc_buffer, echo_read); } else { uv_timer_stop(pConn->pTimer); - free(pConn->pTimer); + taosMemoryFree(pConn->pTimer); uv_close((uv_handle_t *)pConn->pClient, NULL); - free(pConn->pClient); - free(pConn); + taosMemoryFree(pConn->pClient); + taosMemoryFree(pConn); } } @@ -144,13 +144,13 @@ static void workerAsyncCallback(uv_async_t *handle) { void *worker_thread(void *arg) { SThreadObj *pObj = (SThreadObj *)arg; int fd = pObj->fd; - pObj->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + pObj->loop = (uv_loop_t *)taosMemoryMalloc(sizeof(uv_loop_t)); uv_loop_init(pObj->loop); uv_pipe_init(pObj->loop, pObj->pipe, 1); uv_pipe_open(pObj->pipe, fd); - pObj->workerAsync = malloc(sizeof(uv_async_t)); + pObj->workerAsync = taosMemoryMalloc(sizeof(uv_async_t)); uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCallback); uv_read_start((uv_stream_t *)pObj->pipe, alloc_buffer, child_on_new_connection); @@ -159,19 +159,19 @@ void *worker_thread(void *arg) { } int main() { - SServerObj *server = calloc(1, sizeof(SServerObj)); - server->loop = (uv_loop_t *)malloc(sizeof(uv_loop_t)); + SServerObj *server = taosMemoryCalloc(1, sizeof(SServerObj)); + server->loop = (uv_loop_t *)taosMemoryMalloc(sizeof(uv_loop_t)); server->numOfThread = NUM_OF_THREAD; server->workerIdx = 0; server->pThreadObj = - (SThreadObj **)calloc(server->numOfThread, sizeof(SThreadObj *)); - server->pipe = (uv_pipe_t **)calloc(server->numOfThread, sizeof(uv_pipe_t *)); + (SThreadObj **)taosMemoryCalloc(server->numOfThread, sizeof(SThreadObj *)); + server->pipe = (uv_pipe_t **)taosMemoryCalloc(server->numOfThread, sizeof(uv_pipe_t *)); uv_loop_init(server->loop); for (int i = 0; i < server->numOfThread; i++) { - server->pThreadObj[i] = (SThreadObj *)calloc(1, sizeof(SThreadObj)); - server->pipe[i] = (uv_pipe_t *)calloc(2, sizeof(uv_pipe_t)); + server->pThreadObj[i] = (SThreadObj *)taosMemoryCalloc(1, sizeof(SThreadObj)); + server->pipe[i] = (uv_pipe_t *)taosMemoryCalloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 70ec7e0655..83c48628a3 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -37,8 +37,7 @@ void* tmemmem(char* haystack, int hlen, char* needle, int nlen) { } limit = haystack + hlen - nlen + 1; - while ((haystack = (char*)memchr( - haystack, needle[0], limit - haystack)) != NULL) { + while ((haystack = (char*)memchr(haystack, needle[0], limit - haystack)) != NULL) { if (memcmp(haystack, needle, nlen) == 0) { return haystack; } @@ -57,8 +56,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { } #endif - SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz-1); - char fnameStr[WAL_FILE_LEN]; + SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); + char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); int64_t file_size = 0; @@ -74,7 +73,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { uint64_t magic = WAL_MAGIC; - char* buf = malloc(readSize + 5); + char* buf = taosMemoryMalloc(readSize + 5); if (buf == NULL) { taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; @@ -83,35 +82,35 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { taosLSeekFile(pFile, -readSize, SEEK_END); if (readSize != taosReadFile(pFile, buf, readSize)) { - free(buf); + taosMemoryFree(buf); taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - + char* haystack = buf; char* found = NULL; - char *candidate; - while((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { + char* candidate; + while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { // read and validate - SWalHead *logContent = (SWalHead*)candidate; + SWalHead* logContent = (SWalHead*)candidate; if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { found = candidate; } haystack = candidate + 1; } if (found == buf) { - SWalHead *logContent = (SWalHead*)found; + SWalHead* logContent = (SWalHead*)found; if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { // file has to be deleted - free(buf); + taosMemoryFree(buf); taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } } taosCloseFile(&pFile); - SWalHead *lastEntry = (SWalHead*)found; + SWalHead* lastEntry = (SWalHead*)found; return lastEntry->head.version; } @@ -158,10 +157,10 @@ int walCheckAndRepairMeta(SWal* pWal) { int newSz = taosArrayGetSize(pLogInfoArray); if (oldSz > newSz) { - taosArrayPopFrontBatch(pWal->fileInfoSet, oldSz - newSz); + taosArrayPopFrontBatch(pWal->fileInfoSet, oldSz - newSz); } else if (oldSz < newSz) { for (int i = oldSz; i < newSz; i++) { - SWalFileInfo *pFileInfo = taosArrayGet(pLogInfoArray, i); + SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, i); taosArrayPush(pWal->fileInfoSet, pFileInfo); } } @@ -171,8 +170,8 @@ int walCheckAndRepairMeta(SWal* pWal) { if (newSz > 0) { pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; - SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz-1); - char fnameStr[WAL_FILE_LEN]; + SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz - 1); + char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); int64_t file_size = 0; taosStatFile(fnameStr, &file_size, NULL); @@ -191,8 +190,8 @@ int walCheckAndRepairMeta(SWal* pWal) { } } - //TODO: set fileSize and lastVer if necessary - + // TODO: set fileSize and lastVer if necessary + return 0; } @@ -215,7 +214,7 @@ int walRollFileInfo(SWal* pWal) { } // TODO: change to emplace back - SWalFileInfo* pNewInfo = malloc(sizeof(SWalFileInfo)); + SWalFileInfo* pNewInfo = taosMemoryMalloc(sizeof(SWalFileInfo)); if (pNewInfo == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; @@ -226,7 +225,7 @@ int walRollFileInfo(SWal* pWal) { pNewInfo->closeTs = -1; pNewInfo->fileSize = 0; taosArrayPush(pArray, pNewInfo); - free(pNewInfo); + taosMemoryFree(pNewInfo); return 0; } @@ -239,13 +238,13 @@ char* walMetaSerialize(SWal* pWal) { cJSON* pFiles = cJSON_CreateArray(); cJSON* pField; if (pRoot == NULL || pMeta == NULL || pFiles == NULL) { - if(pRoot) { + if (pRoot) { cJSON_Delete(pRoot); } - if(pMeta) { + if (pMeta) { cJSON_Delete(pMeta); } - if(pFiles) { + if (pFiles) { cJSON_Delete(pFiles); } terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; @@ -378,7 +377,7 @@ int walSaveMeta(SWal* pWal) { walBuildMetaName(pWal, metaVer, fnameStr); taosRemoveFile(fnameStr); } - free(serialized); + taosMemoryFree(serialized); return 0; } @@ -395,7 +394,7 @@ int walLoadMeta(SWal* pWal) { int64_t file_size = 0; taosStatFile(fnameStr, &file_size, NULL); int size = (int)file_size; - char* buf = malloc(size + 5); + char* buf = taosMemoryMalloc(size + 5); if (buf == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; @@ -409,12 +408,12 @@ int walLoadMeta(SWal* pWal) { if (taosReadFile(pFile, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - free(buf); + taosMemoryFree(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); taosCloseFile(&pFile); - free(buf); + taosMemoryFree(buf); return code; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 2da8f4f8af..cf40c998de 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -63,7 +63,7 @@ void walCleanUp() { } SWal *walOpen(const char *path, SWalCfg *pCfg) { - SWal *pWal = malloc(sizeof(SWal)); + SWal *pWal = taosMemoryMalloc(sizeof(SWal)); if (pWal == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -88,7 +88,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -103,7 +103,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -111,7 +111,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { if (pWal->refId < 0) { taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -121,7 +121,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { taosRemoveRef(tsWal.refSetId, pWal->refId); taosThreadMutexDestroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); - free(pWal); + taosMemoryFree(pWal); return NULL; } @@ -174,7 +174,7 @@ static void walFreeObj(void *wal) { wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); taosThreadMutexDestroy(&pWal->mutex); - tfree(pWal); + taosMemoryFreeClear(pWal); } static bool walNeedFsync(SWal *pWal) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 8d3acef9e7..5296a16703 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "walInt.h" #include "taoserror.h" +#include "walInt.h" SWalReadHandle *walOpenReadHandle(SWal *pWal) { - SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); + SWalReadHandle *pRead = taosMemoryMalloc(sizeof(SWalReadHandle)); if (pRead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -30,10 +30,10 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { pRead->curFileFirstVer = -1; pRead->capacity = 0; pRead->status = 0; - pRead->pHead = malloc(sizeof(SWalHead)); + pRead->pHead = taosMemoryMalloc(sizeof(SWalHead)); if (pRead->pHead == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - free(pRead); + taosMemoryFree(pRead); return NULL; } return pRead; @@ -42,8 +42,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { void walCloseReadHandle(SWalReadHandle *pRead) { taosCloseFile(&pRead->pReadIdxTFile); taosCloseFile(&pRead->pReadLogTFile); - tfree(pRead->pHead); - free(pRead); + taosMemoryFreeClear(pRead->pHead); + taosMemoryFree(pRead); } int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } @@ -92,6 +92,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pIdxTFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -152,11 +153,12 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } code = walValidHeadCksum(pRead->pHead); if (code != 0) { + wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } if (pRead->capacity < pRead->pHead->head.len) { - void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); + void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); if (ptr == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; @@ -169,7 +171,8 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } if (pRead->pHead->head.version != ver) { - wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver); + wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, + ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -177,7 +180,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { code = walValidBodyCksum(pRead->pHead); if (code != 0) { - wError("unexpected wal log version: checksum not passed"); + wError("unexpected wal log version: % " PRId64 ", since body checksum not passed", ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -195,7 +198,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { return code; } if (*ppHead == NULL) { - void *ptr = realloc(*ppHead, sizeof(SWalHead)); + void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead)); if (ptr == NULL) { return -1; } @@ -208,9 +211,9 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { if (walValidHeadCksum(*ppHead) != 0) { return -1; } - void *ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); + void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); if (ptr == NULL) { - free(*ppHead); + taosMemoryFree(*ppHead); *ppHead = NULL; return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 8392e66e58..a578c6f368 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -253,7 +253,8 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { return 0; } -int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) { +int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body, + int32_t bodyLen) { if (pWal == NULL) return -1; int code = 0; @@ -296,6 +297,10 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in int64_t offset = walGetCurFileOffset(pWal); pWal->writeHead.head.len = bodyLen; pWal->writeHead.head.msgType = msgType; + + // sync info + pWal->writeHead.head.syncMeta = syncMeta; + pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); @@ -332,6 +337,15 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in return 0; } +int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) { + SSyncLogMeta syncMeta = { + .isWeek = -1, + .seqNum = UINT64_MAX, + .term = UINT64_MAX, + }; + return walWriteWithSyncInfo(pWal, index, msgType, syncMeta, body, bodyLen); +} + void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 69ae83154a..b1de3a1dce 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -19,7 +19,7 @@ class WalCleanEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; @@ -27,7 +27,7 @@ class WalCleanEnv : public ::testing::Test { pCfg->retentionSize = 0; pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); - free(pCfg); + taosMemoryFree(pCfg); ASSERT(pWal != NULL); } @@ -51,13 +51,13 @@ class WalCleanDeleteEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->retentionPeriod = 0; pCfg->retentionSize = 0; pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); - free(pCfg); + taosMemoryFree(pCfg); ASSERT(pWal != NULL); } @@ -86,7 +86,7 @@ class WalKeepEnv : public ::testing::Test { } void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; @@ -94,7 +94,7 @@ class WalKeepEnv : public ::testing::Test { pCfg->retentionSize = 0; pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); - free(pCfg); + taosMemoryFree(pCfg); ASSERT(pWal != NULL); } @@ -172,7 +172,7 @@ TEST_F(WalCleanEnv, serialize) { ASSERT(code == 0); char* ss = walMetaSerialize(pWal); printf("%s\n", ss); - free(ss); + taosMemoryFree(ss); code = walSaveMeta(pWal); ASSERT(code == 0); } @@ -216,8 +216,8 @@ TEST_F(WalKeepEnv, readOldMeta) { for (int i = 0; i < len; i++) { EXPECT_EQ(oldss[i], newss[i]); } - free(oldss); - free(newss); + taosMemoryFree(oldss); + taosMemoryFree(newss); } TEST_F(WalCleanEnv, write) { diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 61b2593bc6..22884298ef 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -18,30 +18,31 @@ extern void taosWinSocketInit(); -char configDir[PATH_MAX] = {0}; -char tsDataDir[PATH_MAX] = {0}; -char tsLogDir[PATH_MAX] = {0}; -char tsTempDir[PATH_MAX] = {0}; -SDiskSpace tsDataSpace = {0}; -SDiskSpace tsLogSpace = {0}; -SDiskSpace tsTempSpace = {0}; -char tsOsName[16] = {0}; -char tsTimezone[TD_TIMEZONE_LEN] = {0}; -char tsLocale[TD_LOCALE_LEN] = {0}; -char tsCharset[TD_CHARSET_LEN] = {0}; -int8_t tsDaylight = 0; -bool tsEnableCoreFile = 0; -int64_t tsPageSizeKB = 0; -int64_t tsOpenMax = 0; -int64_t tsStreamMax = 0; -float tsNumOfCores = 0; -int64_t tsTotalMemoryKB = 0; +char configDir[PATH_MAX] = {0}; +char tsDataDir[PATH_MAX] = {0}; +char tsLogDir[PATH_MAX] = {0}; +char tsTempDir[PATH_MAX] = {0}; +SDiskSpace tsDataSpace = {0}; +SDiskSpace tsLogSpace = {0}; +SDiskSpace tsTempSpace = {0}; +char tsOsName[16] = {0}; +char tsTimezoneStr[TD_TIMEZONE_LEN] = {0}; +enum TdTimezone tsTimezone = TdZeroZone; +char tsLocale[TD_LOCALE_LEN] = {0}; +char tsCharset[TD_CHARSET_LEN] = {0}; +int8_t tsDaylight = 0; +bool tsEnableCoreFile = 0; +int64_t tsPageSizeKB = 0; +int64_t tsOpenMax = 0; +int64_t tsStreamMax = 0; +float tsNumOfCores = 0; +int64_t tsTotalMemoryKB = 0; -void osInit() { +void osDefaultInit() { taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); - taosGetSystemTimezone(tsTimezone); - taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); + taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); + taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone); taosGetSystemInfo(); // deadlock in query @@ -105,4 +106,9 @@ void osCleanup() {} bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } -void osSetTimezone(const char *timezone) { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); } +void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); } + +void osSetSystemLocale(const char *inLocale, const char *inCharSet) { + memcpy(tsLocale, inLocale, strlen(inLocale) + 1); + memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1); +} diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4dece8abef..ed93708c07 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -268,7 +268,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { autoDelFileListAdd(path); } - TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile)); + TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile)); if (pFile == NULL) { if (fd >= 0) close(fd); if (fp != NULL) fclose(fp); @@ -312,7 +312,7 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { taosThreadRwlockUnlock(&((*ppFile)->rwlock)); taosThreadRwlockDestroy(&((*ppFile)->rwlock)); #endif - free(*ppFile); + taosMemoryFree(*ppFile); *ppFile = NULL; return 0; #endif diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 5f12f9cd3d..2df61c08d4 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -134,7 +134,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { char *revisedCharset = taosCharsetReplace(str); tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); - free(revisedCharset); + taosMemoryFree(revisedCharset); // printf("charset not configured, set to system default:%s", outCharset); } else { strcpy(outCharset, "UTF-8"); @@ -179,7 +179,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { char *revisedCharset = taosCharsetReplace(str); tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN); - free(revisedCharset); + taosMemoryFree(revisedCharset); // printf("charset not configured, set to system default:%s", outCharset); } else { strcpy(outCharset, "UTF-8"); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c new file mode 100644 index 0000000000..12e89fdd73 --- /dev/null +++ b/source/os/src/osMemory.c @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC +#include "os.h" + +#define TD_MEMORY_SYMBOL ('T'<<24|'A'<<16|'O'<<8|'S') + +#define TD_MEMORY_STACK_TRACE_DEPTH 10 + +typedef struct TdMemoryInfo +{ + int32_t symbol; + void *stackTrace[TD_MEMORY_STACK_TRACE_DEPTH]; // gdb: disassemble /m 0xXXX + int32_t memorySize; +} *TdMemoryInfoPtr , TdMemoryInfo; + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +#else + +#include +#define STACKCALL __attribute__((regparm(1), noinline)) +void **STACKCALL taosGetEbp(void) { + void **ebp = NULL; + __asm__ __volatile__("mov %%rbp, %0;\n\t" + : "=m"(ebp) /* output */ + : /* input */ + : "memory"); /* not affect register */ + return (void **)(*ebp); +} +int32_t taosBackTrace(void **buffer, int32_t size) { + int32_t frame = 0; + void **ebp; + void **ret = NULL; + size_t func_frame_distance = 0; + if (buffer != NULL && size > 0) { + ebp = taosGetEbp(); + func_frame_distance = (size_t)*ebp - (size_t)ebp; + while (ebp && frame < size && (func_frame_distance < (1ULL << 24)) // assume function ebp more than 16M + && (func_frame_distance > 0)) { + ret = ebp + 1; + buffer[frame++] = *ret; + ebp = (void **)(*ebp); + func_frame_distance = (size_t)*ebp - (size_t)ebp; + } + } + return frame; +} +#endif + +// char **taosBackTraceSymbols(int32_t *size) { +// void *buffer[20] = {NULL}; +// *size = taosBackTrace(buffer, 20); +// return backtrace_symbols(buffer, *size); +// } + +void *taosMemoryMalloc(int32_t size) { + void *tmp = malloc(size + sizeof(TdMemoryInfo)); + if (tmp == NULL) return NULL; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)tmp; + pTdMemoryInfo->memorySize = size; + pTdMemoryInfo->symbol = TD_MEMORY_SYMBOL; + taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH); + + return (char*)tmp + sizeof(TdMemoryInfo); +} + +void *taosMemoryCalloc(int32_t num, int32_t size) { + int32_t memorySize = num * size; + char *tmp = calloc(memorySize + sizeof(TdMemoryInfo), 1); + if (tmp == NULL) return NULL; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)tmp; + pTdMemoryInfo->memorySize = memorySize; + pTdMemoryInfo->symbol = TD_MEMORY_SYMBOL; + taosBackTrace(pTdMemoryInfo->stackTrace,TD_MEMORY_STACK_TRACE_DEPTH); + + return (char*)tmp + sizeof(TdMemoryInfo); +} + +void *taosMemoryRealloc(void *ptr, int32_t size) { + if (ptr == NULL) return taosMemoryMalloc(size); + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + + TdMemoryInfo tdMemoryInfo; + memcpy(&tdMemoryInfo, pTdMemoryInfo, sizeof(TdMemoryInfo)); + + void *tmp = realloc(pTdMemoryInfo, size + sizeof(TdMemoryInfo)); + if (tmp == NULL) return NULL; + + memcpy(tmp, &tdMemoryInfo, sizeof(TdMemoryInfo)); + ((TdMemoryInfoPtr)tmp)->memorySize = size; + + return (char*)tmp + sizeof(TdMemoryInfo); +} + +void taosMemoryFree(const void *ptr) { + if (ptr == NULL) return; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + if(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL) { + pTdMemoryInfo->memorySize = 0; + // memset(pTdMemoryInfo, 0, sizeof(TdMemoryInfo)); + free(pTdMemoryInfo); + } else { + free((void*)ptr); + } +} + +int32_t taosMemorySize(void *ptr) { + if (ptr == NULL) return 0; + + TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); + assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + + return pTdMemoryInfo->memorySize; +} diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index b41ef898d4..1297fdbc27 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -143,7 +143,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) { fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); } - struct tsem_s *p = (struct tsem_s *)calloc(1, sizeof(*p)); + struct tsem_s *p = (struct tsem_s *)taosMemoryCalloc(1, sizeof(*p)); if (!p) { fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); abort(); @@ -331,7 +331,7 @@ int tsem_destroy(tsem_t *sem) { #endif // SEM_USE_PTHREAD p->valid = 0; - free(p); + taosMemoryFree(p); *sem = NULL; return 0; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index f693ad74de..b130689d4b 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -117,7 +117,7 @@ int32_t taosCloseSocket(TdSocketPtr *ppSocket) { } code = taosCloseSocketNoCheck1((*ppSocket)->fd); (*ppSocket)->fd = -1; - free(*ppSocket); + taosMemoryFree(*ppSocket); return code; } int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { @@ -127,7 +127,7 @@ int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer) { } code = taosCloseSocketNoCheck1((*ppSocketServer)->fd); (*ppSocketServer)->fd = -1; - free(*ppSocketServer); + taosMemoryFree(*ppSocketServer); return code; } @@ -440,7 +440,7 @@ TdSocketPtr taosOpenUdpSocket(uint32_t ip, uint16_t port) { return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -484,7 +484,7 @@ TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -654,7 +654,7 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -703,7 +703,7 @@ TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct s return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -901,7 +901,7 @@ TdEpollPtr taosCreateEpoll(int32_t size) { return NULL; } - TdEpollPtr pEpoll = (TdEpollPtr)malloc(sizeof(TdEpoll)); + TdEpollPtr pEpoll = (TdEpollPtr)taosMemoryMalloc(sizeof(TdEpoll)); if (pEpoll == NULL) { taosCloseSocketNoCheck1(fd); return NULL; @@ -939,6 +939,6 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) { } code = taosCloseSocketNoCheck1((*ppEpoll)->fd); (*ppEpoll)->fd = -1; - free(*ppEpoll); + taosMemoryFree(*ppEpoll); return code; } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index d3d1ab5dda..5adb564e52 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -15,7 +15,6 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE -#include #include "os.h" #ifndef DISALLOW_NCHAR_WITHOUT_ICONV @@ -50,8 +49,8 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { #if 0 int32_t ucs4_max_len = bytes + 4; - char *f1_mbs = calloc(bytes, 1); - char *f2_mbs = calloc(bytes, 1); + char *f1_mbs = taosMemoryCalloc(bytes, 1); + char *f2_mbs = taosMemoryCalloc(bytes, 1); if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { return -1; } @@ -59,15 +58,15 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { return -1; } int32_t ret = strcmp(f1_mbs, f2_mbs); - free(f1_mbs); - free(f2_mbs); + taosMemoryFree(f1_mbs); + taosMemoryFree(f2_mbs); return ret; #endif } TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { - assert(malloc_usable_size(target_ucs4)>=len_ucs4*sizeof(TdUcs4)); + assert(taosMemorySize(target_ucs4)>=len_ucs4*sizeof(TdUcs4)); return memcpy(target_ucs4, source_ucs4, len_ucs4*sizeof(TdUcs4)); } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 9bff760509..eb5712d2e5 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -145,7 +145,7 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; } @@ -174,7 +174,7 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; } @@ -268,7 +268,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; #else @@ -293,7 +293,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; #endif @@ -324,7 +324,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; @@ -351,7 +351,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { } } - if (line != NULL) free(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); return code; @@ -486,7 +486,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) { char tmp[10]; sscanf(line, "%s %" PRId64, tmp, usedKB); - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; #endif @@ -606,7 +606,7 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int if (readIndex >= 4) break; } - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); if (readIndex < 4) { @@ -665,7 +665,7 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { *transmit_bytes = o_tbytes; } - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); return 0; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index bdafa63d64..dc23eaae1a 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -46,12 +46,22 @@ #include #endif -void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { - if (inTimezone == NULL || inTimezone[0] == 0) return; +void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8_t *outDaylight, enum TdTimezone *tsTimezone) { + if (inTimezoneStr == NULL || inTimezoneStr[0] == 0) return; + + char *buf = taosMemoryMalloc(strlen(inTimezoneStr) + 1); + buf[strlen(inTimezoneStr)] = 0; + for (int32_t i = 0; i < strlen(inTimezoneStr); i++) { + if(inTimezoneStr[i]==' ' || inTimezoneStr[i]=='(') { + buf[i] = 0; + break; + } + buf[i] = inTimezoneStr[i]; + } #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) char winStr[TD_LOCALE_LEN * 2]; - sprintf(winStr, "TZ=%s", inTimezone); + sprintf(winStr, "TZ=%s", buf); putenv(winStr); tzset(); * get CURRENT time zone. @@ -70,44 +80,48 @@ void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *ou #endif int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + *tsTimezone = tz; tz += daylight; /* * format: * (CST, +0800) * (BST, +0100) */ - sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = daylight; #elif defined(_TD_DARWIN_64) - setenv("TZ", inTimezone, 1); + setenv("TZ", buf, 1); tzset(); int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + *tsTimezone = tz; tz += daylight; - sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = daylight; #else - setenv("TZ", inTimezone, 1); + setenv("TZ", buf, 1); tzset(); int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + *tsTimezone = tz; tz += daylight; - sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = daylight; #endif + taosMemoryFree(buf); } -void taosGetSystemTimezone(char *outTimezone) { +void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) char *tz = getenv("TZ"); if (tz == NULL || strlen(tz) == 0) { - strcpy(outTimezone, "not configured"); + strcpy(outTimezoneStr, "not configured"); } else { - strcpy(outTimezone, tz); + strcpy(outTimezoneStr, tz); } #elif defined(_TD_DARWIN_64) @@ -153,7 +167,7 @@ void taosGetSystemTimezone(char *outTimezone) { * Asia/Shanghai (CST, +0800) * Europe/London (BST, +0100) */ - snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); #else @@ -168,13 +182,14 @@ void taosGetSystemTimezone(char *outTimezone) { /* load time zone string from /etc/timezone */ // FILE *f = fopen("/etc/timezone", "r"); + errno = 0; TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); char buf[68] = {0}; if (pFile != NULL) { int len = taosReadFile(pFile, buf, 64); if (len < 64 && taosGetErrorFile(pFile)) { taosCloseFile(&pFile); - // printf("read /etc/timezone error, reason:%s", strerror(errno)); + printf("read /etc/timezone error, reason:%s", strerror(errno)); return; } @@ -202,6 +217,7 @@ void taosGetSystemTimezone(char *outTimezone) { * otherwise is GMT+00:00 */ int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + *tsTimezone = tz; tz += daylight; /* @@ -210,7 +226,7 @@ void taosGetSystemTimezone(char *outTimezone) { * Asia/Shanghai (CST, +0800) * Europe/London (BST, +0100) */ - snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); #endif } diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index cb0a2d25a9..48a0f327c4 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -153,9 +153,9 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, cons } void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { - char *buf = calloc(1, size); // prepare the swap buffer + char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); - tfree(buf); + taosMemoryFreeClear(buf); } void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t compar, int32_t flags) { @@ -239,7 +239,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const child = 2 * parent + 1; if (swap == NULL) { - buf = calloc(1, size); + buf = taosMemoryCalloc(1, size); if (buf == NULL) { return; } @@ -288,7 +288,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const } if (swap == NULL) { - tfree(buf); + taosMemoryFreeClear(buf); } } } @@ -304,13 +304,13 @@ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, } /* - char *buf = calloc(1, size); + char *buf = taosMemoryCalloc(1, size); for (i = len - 1; i > 0; i--) { doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i)); taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot); } - tfree(buf); + taosMemoryFreeClear(buf); */ } diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 1127a21255..a74b26a386 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -23,15 +23,15 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { size = TARRAY_MIN_SIZE; } - SArray* pArray = malloc(sizeof(SArray)); + SArray* pArray = taosMemoryMalloc(sizeof(SArray)); if (pArray == NULL) { return NULL; } pArray->size = 0; - pArray->pData = calloc(size, elemSize); + pArray->pData = taosMemoryCalloc(size, elemSize); if (pArray->pData == NULL) { - free(pArray); + taosMemoryFree(pArray); return NULL; } @@ -46,7 +46,7 @@ static int32_t taosArrayResize(SArray* pArray) { size_t size = pArray->capacity; size = (size << 1u); - void* tmp = realloc(pArray->pData, size * pArray->elemSize); + void* tmp = taosMemoryRealloc(pArray->pData, size * pArray->elemSize); if (tmp == NULL) { // reallocate failed, the original buffer remains return -1; } @@ -64,7 +64,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { tsize = (tsize << 1u); } - pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize); + pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); if (pArray->pData == NULL) { return -1; } @@ -305,8 +305,8 @@ void taosArrayClear(SArray* pArray) { void* taosArrayDestroy(SArray* pArray) { if (pArray) { - free(pArray->pData); - free(pArray); + taosMemoryFree(pArray->pData); + taosMemoryFree(pArray); } return NULL; diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index c5119e8b2d..a2f4ddbc51 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -20,7 +20,7 @@ static char basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01 char *base64_encode(const uint8_t *value, int32_t vlen) { uint8_t oval = 0; - char *result = (char *)malloc((size_t)(vlen * 4) / 3 + 10); + char *result = (char *)taosMemoryMalloc((size_t)(vlen * 4) / 3 + 10); char *out = result; while (vlen >= 3) { *out++ = basis_64[value[0] >> 2]; @@ -53,7 +53,7 @@ static signed char index_64[128] = { uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { int32_t c1, c2, c3, c4; - uint8_t *result = (uint8_t *)malloc((size_t)(inlen * 3) / 4 + 1); + uint8_t *result = (uint8_t *)taosMemoryMalloc((size_t)(inlen * 3) / 4 + 1); uint8_t *out = result; *outlen = 0; @@ -93,7 +93,7 @@ uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { } base64_decode_error: - free(result); + taosMemoryFree(result); result = 0; *outlen = 0; diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index dc8c4b70c5..8552ccac2c 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -213,7 +213,7 @@ double tbufReadDouble(SBufferReader* buf) { // writer functions void tbufCloseWriter(SBufferWriter* buf) { - tfree(buf->data); + taosMemoryFreeClear(buf->data); // (*buf->allocator)( buf->data, 0 ); // potential memory leak. buf->data = NULL; buf->pos = 0; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index cf5b623678..c4e85a425e 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -221,7 +221,7 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode * pCacheObj->freeFp(pNode->data); } - free(pNode); + taosMemoryFree(pNode); } static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STrashElem *pElem) { @@ -255,8 +255,8 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem pCacheObj->freeFp(pElem->pData->data); } - free(pElem->pData); - free(pElem); + taosMemoryFree(pElem->pData); + taosMemoryFree(pElem); } static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { @@ -358,7 +358,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - SCacheObj *pCacheObj = (SCacheObj *)calloc(1, sizeof(SCacheObj)); + SCacheObj *pCacheObj = (SCacheObj *)taosMemoryCalloc(1, sizeof(SCacheObj)); if (pCacheObj == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; @@ -366,9 +366,9 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext // TODO add the auto extend procedure pCacheObj->capacity = 4096; - pCacheObj->pEntryList = calloc(pCacheObj->capacity, sizeof(SCacheEntry)); + pCacheObj->pEntryList = taosMemoryCalloc(pCacheObj->capacity, sizeof(SCacheEntry)); if (pCacheObj->pEntryList == NULL) { - free(pCacheObj); + taosMemoryFree(pCacheObj); uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } @@ -381,8 +381,8 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext pCacheObj->extendLifespan = extendLifespan; // the TTL after the last access if (__trashcan_lock_init(pCacheObj) != 0) { - tfree(pCacheObj->pEntryList); - free(pCacheObj); + taosMemoryFreeClear(pCacheObj->pEntryList); + taosMemoryFree(pCacheObj); uError("failed to init lock, reason:%s", strerror(errno)); return NULL; @@ -432,7 +432,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v } atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); - tfree(pNode); + taosMemoryFreeClear(pNode); } else { taosAddToTrashcan(pCacheObj, pNode); uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pNode->data); @@ -625,7 +625,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { pCacheObj->freeFp(pNode->data); } - free(pNode); + taosMemoryFree(pNode); } } @@ -703,7 +703,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { size_t sizeInBytes = size + sizeof(SCacheNode) + keyLen; - SCacheNode *pNewNode = calloc(1, sizeInBytes); + SCacheNode *pNewNode = taosMemoryCalloc(1, sizeInBytes); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; uError("failed to allocate memory, reason:%s", strerror(errno)); @@ -735,7 +735,7 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) { } __trashcan_wr_lock(pCacheObj); - STrashElem *pElem = calloc(1, sizeof(STrashElem)); + STrashElem *pElem = taosMemoryCalloc(1, sizeof(STrashElem)); pElem->pData = pNode; pElem->prev = NULL; pElem->next = NULL; @@ -802,9 +802,9 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { __trashcan_lock_destroy(pCacheObj); - tfree(pCacheObj->pEntryList); - tfree(pCacheObj->name); - free(pCacheObj); + taosMemoryFreeClear(pCacheObj->pEntryList); + taosMemoryFreeClear(pCacheObj->name); + taosMemoryFree(pCacheObj); } static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) { @@ -918,7 +918,7 @@ size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj) { SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj) { ASSERT(pCacheObj != NULL); - SCacheIter* pIter = calloc(1, sizeof(SCacheIter)); + SCacheIter* pIter = taosMemoryCalloc(1, sizeof(SCacheIter)); pIter->pCacheObj = (SCacheObj*) pCacheObj; pIter->entryIndex = -1; pIter->index = -1; @@ -955,7 +955,7 @@ bool taosCacheIterNext(SCacheIter* pIter) { } if (pIter->numOfObj < pEntry->num) { - char *tmp = realloc(pIter->pCurrent, pEntry->num * POINTER_BYTES); + char *tmp = taosMemoryRealloc(pIter->pCurrent, pEntry->num * POINTER_BYTES); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosRUnLockLatch(&pEntry->latch); @@ -1001,6 +1001,6 @@ void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* len) { } void taosCacheDestroyIter(SCacheIter* pIter) { - tfree(pIter->pCurrent); - tfree(pIter); + taosMemoryFreeClear(pIter->pCurrent); + taosMemoryFreeClear(pIter); } \ No newline at end of file diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index ff7d2cf733..c98f6eb9be 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -355,12 +355,12 @@ int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight) { int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { size_t sz = varDataLen(pRight); - char *pattern = malloc(sz + 1); + char *pattern = taosMemoryMalloc(sz + 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); pattern[sz] = 0; sz = varDataLen(pLeft); - char *str = malloc(sz + 1); + char *str = taosMemoryMalloc(sz + 1); memcpy(str, varDataVal(pLeft), sz); str[sz] = 0; @@ -373,8 +373,8 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf); regfree(®ex); - free(str); - free(pattern); + taosMemoryFree(str); + taosMemoryFree(pattern); return 1; } @@ -385,8 +385,8 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { } int32_t result = (errCode == 0) ? 0 : 1; regfree(®ex); - free(str); - free(pattern); + taosMemoryFree(str); + taosMemoryFree(pattern); return result; } @@ -401,17 +401,17 @@ int32_t compareStrPatternMatch(const void *pLeft, const void *pRight) { SPatternCompareInfo pInfo = {'%', '_'}; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); - char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); + char *pattern = taosMemoryCalloc(varDataLen(pRight) + 1, sizeof(char)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); size_t sz = varDataLen(pLeft); - char *buf = malloc(sz + 1); + char *buf = taosMemoryMalloc(sz + 1); memcpy(buf, varDataVal(pLeft), sz); buf[sz] = 0; int32_t ret = patternMatch(pattern, buf, sz, &pInfo); - free(buf); - free(pattern); + taosMemoryFree(buf); + taosMemoryFree(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } @@ -424,11 +424,11 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); - char *pattern = calloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); + char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); int32_t ret = WCSPatternMatch((TdUcs4*)pattern, (TdUcs4*)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); - free(pattern); + taosMemoryFree(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 404c9c8f71..96bb638b95 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -29,7 +29,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); SConfig *cfgInit() { - SConfig *pCfg = calloc(1, sizeof(SConfig)); + SConfig *pCfg = taosMemoryCalloc(1, sizeof(SConfig)); if (pCfg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -37,7 +37,7 @@ SConfig *cfgInit() { pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); if (pCfg->array == NULL) { - free(pCfg); + taosMemoryFree(pCfg); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -75,7 +75,7 @@ int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs) { static void cfgFreeItem(SConfigItem *pItem) { if (pItem->dtype == CFG_DTYPE_STRING || pItem->dtype == CFG_DTYPE_DIR || pItem->dtype == CFG_DTYPE_LOCALE || pItem->dtype == CFG_DTYPE_CHARSET || pItem->dtype == CFG_DTYPE_TIMEZONE) { - tfree(pItem->str); + taosMemoryFreeClear(pItem->str); } if (pItem->array) { taosArrayDestroy(pItem->array); @@ -89,10 +89,10 @@ void cfgCleanup(SConfig *pCfg) { for (int32_t i = 0; i < size; ++i) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); cfgFreeItem(pItem); - tfree(pItem->name); + taosMemoryFreeClear(pItem->name); } taosArrayDestroy(pCfg->array); - free(pCfg); + taosMemoryFree(pCfg); } } @@ -145,7 +145,7 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } - tfree(pItem->str); + taosMemoryFreeClear(pItem->str); pItem->str = strdup(fullDir); if (pItem->str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -224,7 +224,7 @@ static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType s return -1; } - free(pItem->str); + taosMemoryFree(pItem->str); pItem->str = tmp; pItem->stype = stype; return 0; @@ -366,9 +366,9 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { if (taosArrayPush(pCfg->array, pItem) == NULL) { if (pItem->dtype == CFG_DTYPE_STRING) { - free(pItem->str); + taosMemoryFree(pItem->str); } - free(pItem->name); + taosMemoryFree(pItem->name); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -647,7 +647,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } taosCloseFile(&pFile); - if (line != NULL) tfree(line); + if (line != NULL) taosMemoryFreeClear(line); uInfo("load from cfg file %s success", filepath); return 0; diff --git a/source/util/src/tcrc32c.c b/source/util/src/tcrc32c.c index f7e1060be0..66e9240cd5 100644 --- a/source/util/src/tcrc32c.c +++ b/source/util/src/tcrc32c.c @@ -19,13 +19,13 @@ */ #define _DEFAULT_SOURCE -#include "tcrc32c.h" -#include "tdef.h" - #if !defined(_TD_ARM_) && !defined(_TD_MIPS_) #include #endif +#include "tcrc32c.h" +#include "tdef.h" + #define POLY 0x82f63b78 #define LONG_SHIFT 8192 #define SHORT_SHIFT 256 diff --git a/source/util/src/tdes.c b/source/util/src/tdes.c index a7f5131c26..0e0193a123 100644 --- a/source/util/src/tdes.c +++ b/source/util/src/tdes.c @@ -47,7 +47,7 @@ char* taosDesImp(uint8_t* key, char* src, uint32_t len, int32_t process_mode) { uint8_t processed_block[9] = {0}; key_set key_sets[17]; memset(key_sets, 0, sizeof(key_sets)); - char* dest = calloc(len + 1, 1); + char* dest = taosMemoryCalloc(len + 1, 1); generate_sub_keys(key, key_sets); for (uint32_t block_count = 0; block_count < number_of_blocks; block_count++) { @@ -68,12 +68,12 @@ char* taosDesEncode(int64_t key, char* src, int32_t len) { char* taosDesDecode(int64_t key, char* src, int32_t len) { uint8_t* keyStr = (uint8_t*)(&key); - char* temp = calloc(len + 8, 1); + char* temp = taosMemoryCalloc(len + 8, 1); memcpy(temp, src, len); len += 8; char* decode = taosDesImp(keyStr, temp, len, DECRYPTION_MODE); - free(temp); + taosMemoryFree(temp); return decode; } diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index 94b4cced46..c40d5b02e6 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -44,7 +44,7 @@ void tCoderClear(SCoder* pCoder) { pNode = TD_SLIST_HEAD(&(pCoder->stack)); if (pNode == NULL) break; TD_SLIST_POP(&(pCoder->stack)); - free(pNode); + taosMemoryFree(pNode); } } @@ -55,7 +55,7 @@ int32_t tStartEncode(SCoder* pCoder) { if (pCoder->data) { if (pCoder->size - pCoder->pos < sizeof(int32_t)) return -1; - pNode = malloc(sizeof(*pNode)); + pNode = taosMemoryMalloc(sizeof(*pNode)); if (pNode == NULL) return -1; pNode->data = pCoder->data; @@ -93,7 +93,7 @@ void tEndEncode(SCoder* pCoder) { TD_CODER_MOVE_POS(pCoder, len); - free(pNode); + taosMemoryFree(pNode); } } @@ -104,7 +104,7 @@ int32_t tStartDecode(SCoder* pCoder) { ASSERT(pCoder->type == TD_DECODER); if (tDecodeI32(pCoder, &len) < 0) return -1; - pNode = malloc(sizeof(*pNode)); + pNode = taosMemoryMalloc(sizeof(*pNode)); if (pNode == NULL) return -1; pNode->data = pCoder->data; @@ -133,5 +133,5 @@ void tEndDecode(SCoder* pCoder) { pCoder->pos = pCoder->size + pNode->pos; pCoder->size = pNode->size; - free(pNode); + taosMemoryFree(pNode); } diff --git a/source/util/src/tfunctional.c b/source/util/src/tfunctional.c index c49fbdb504..3b51d0046f 100644 --- a/source/util/src/tfunctional.c +++ b/source/util/src/tfunctional.c @@ -17,21 +17,21 @@ #include "tfunctional.h" tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int32_t numOfArgs) { - tGenericSavedFunc* pSavedFunc = malloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*))); + tGenericSavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*))); if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int32_t numOfArgs) { - tI32SavedFunc* pSavedFunc = malloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void*)); + tI32SavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void*)); if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int32_t numOfArgs) { - tVoidSavedFunc* pSavedFunc = malloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*)); + tVoidSavedFunc* pSavedFunc = taosMemoryMalloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*)); if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index aac99bfe28..88e5c12770 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -33,7 +33,7 @@ #define FREE_HASH_NODE(_n) \ do { \ - tfree(_n); \ + taosMemoryFreeClear(_n); \ } while (0); struct SHashNode { @@ -238,7 +238,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp capacity = 4; } - SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj)); + SHashObj *pHashObj = (SHashObj *)taosMemoryCalloc(1, sizeof(SHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -254,26 +254,26 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *)); + pHashObj->hashList = (SHashEntry **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { - free(pHashObj); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); if (pHashObj->pMemBlock == NULL) { - free(pHashObj->hashList); - free(pHashObj); + taosMemoryFree(pHashObj->hashList); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); + void *p = taosMemoryCalloc(pHashObj->capacity, sizeof(SHashEntry)); if (p == NULL) { taosArrayDestroy(pHashObj->pMemBlock); - free(pHashObj->hashList); - free(pHashObj); + taosMemoryFree(pHashObj->hashList); + taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -440,14 +440,14 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** if (size != NULL) { if (*d == NULL) { *size = pNode->dataLen; - *d = calloc(1, *size); + *d = taosMemoryCalloc(1, *size); if (*d == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } } else if (*size < pNode->dataLen) { *size = pNode->dataLen; - char* tmp = realloc(*d, *size); + char* tmp = taosMemoryRealloc(*d, *size); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -567,24 +567,24 @@ void taosHashClear(SHashObj *pHashObj) { taosHashWUnlock(pHashObj); } -// the input paras should be SHashObj **, so the origin input will be set by tfree(*pHashObj) +// the input paras should be SHashObj **, so the origin input will be set by taosMemoryFreeClear(*pHashObj) void taosHashCleanup(SHashObj *pHashObj) { if (pHashObj == NULL) { return; } taosHashClear(pHashObj); - tfree(pHashObj->hashList); + taosMemoryFreeClear(pHashObj->hashList); // destroy mem block size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); for (int32_t i = 0; i < memBlock; ++i) { void *p = taosArrayGetP(pHashObj->pMemBlock, i); - tfree(p); + taosMemoryFreeClear(p); } taosArrayDestroy(pHashObj->pMemBlock); - free(pHashObj); + taosMemoryFree(pHashObj); } // for profile only @@ -623,7 +623,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { // uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; @@ -632,7 +632,7 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->hashList = pNewEntryList; size_t inc = newCapacity - pHashObj->capacity; - void * p = calloc(inc, sizeof(SHashEntry)); + void * p = taosMemoryCalloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -683,7 +683,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHashNode *pNewNode = malloc(sizeof(SHashNode) + keyLen + dsize); + SHashNode *pNewNode = taosMemoryMalloc(sizeof(SHashNode) + keyLen + dsize); if (pNewNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/util/src/theap.c b/source/util/src/theap.c index 30af0483cc..8c1a1db057 100644 --- a/source/util/src/theap.c +++ b/source/util/src/theap.c @@ -19,7 +19,7 @@ size_t heapSize(Heap* heap) { return heap->nelts; } Heap* heapCreate(HeapCompareFn fn) { - Heap* heap = calloc(1, sizeof(Heap)); + Heap* heap = taosMemoryCalloc(1, sizeof(Heap)); if (heap == NULL) { return NULL; } @@ -30,7 +30,7 @@ Heap* heapCreate(HeapCompareFn fn) { return heap; } -void heapDestroy(Heap* heap) { free(heap); } +void heapDestroy(Heap* heap) { taosMemoryFree(heap); } HeapNode* heapMin(const Heap* heap) { return heap->min; } diff --git a/source/util/src/tidpool.c b/source/util/src/tidpool.c index 705cb0d2d3..b45113942c 100644 --- a/source/util/src/tidpool.c +++ b/source/util/src/tidpool.c @@ -18,12 +18,12 @@ #include "tlog.h" void *taosInitIdPool(int32_t maxId) { - id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); + id_pool_t *pIdPool = taosMemoryCalloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; - pIdPool->freeList = calloc(maxId, sizeof(bool)); + pIdPool->freeList = taosMemoryCalloc(maxId, sizeof(bool)); if (pIdPool->freeList == NULL) { - free(pIdPool); + taosMemoryFree(pIdPool); return NULL; } @@ -79,13 +79,13 @@ void taosIdPoolCleanUp(id_pool_t *pIdPool) { uDebug("pool:%p is cleaned", pIdPool); - if (pIdPool->freeList) free(pIdPool->freeList); + if (pIdPool->freeList) taosMemoryFree(pIdPool->freeList); taosThreadMutexDestroy(&pIdPool->mutex); memset(pIdPool, 0, sizeof(id_pool_t)); - free(pIdPool); + taosMemoryFree(pIdPool); } int32_t taosIdPoolNumOfUsed(id_pool_t *pIdPool) { @@ -118,7 +118,7 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { return 0; } - bool *idList = calloc(maxId, sizeof(bool)); + bool *idList = taosMemoryCalloc(maxId, sizeof(bool)); if (idList == NULL) { return -1; } @@ -131,7 +131,7 @@ int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { bool *oldIdList = pIdPool->freeList; pIdPool->freeList = idList; - free(oldIdList); + taosMemoryFree(oldIdList); taosThreadMutexUnlock(&pIdPool->mutex); diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index a85da8cbbf..0efcf517a9 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -255,7 +255,7 @@ int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, v if (NULL == pJsonObj) { return TSDB_CODE_FAILED; } - *pObj = calloc(1, objSize); + *pObj = taosMemoryCalloc(1, objSize); if (NULL == *pObj) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 5fccba614b..1d17b4a9e1 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -22,7 +22,7 @@ void tdListInit(SList *list, int32_t eleSize) { } SList *tdListNew(int32_t eleSize) { - SList *list = (SList *)malloc(sizeof(SList)); + SList *list = (SList *)taosMemoryMalloc(sizeof(SList)); if (list == NULL) return NULL; tdListInit(list, eleSize); @@ -33,14 +33,14 @@ void tdListEmpty(SList *list) { SListNode *node; while ((node = TD_DLIST_HEAD(list)) != NULL) { TD_DLIST_POP(list, node); - free(node); + taosMemoryFree(node); } } void *tdListFree(SList *list) { if (list) { tdListEmpty(list); - free(list); + taosMemoryFree(list); } return NULL; @@ -51,7 +51,7 @@ void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, no void tdListAppendNode(SList *list, SListNode *node) { TD_DLIST_APPEND(list, node); } int32_t tdListPrepend(SList *list, void *data) { - SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); + SListNode *node = (SListNode *)taosMemoryMalloc(sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; memcpy((void *)(node->data), data, list->eleSize); @@ -61,7 +61,7 @@ int32_t tdListPrepend(SList *list, void *data) { } int32_t tdListAppend(SList *list, void *data) { - SListNode *node = (SListNode *)calloc(1, sizeof(SListNode) + list->eleSize); + SListNode *node = (SListNode *)taosMemoryCalloc(1, sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; memcpy((void *)(node->data), data, list->eleSize); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 85db7883cf..ef15f44f8f 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -507,10 +507,10 @@ static void taosCloseLogByFd(TdFilePtr pFile) { static SLogBuff *taosLogBuffNew(int32_t bufSize) { SLogBuff *tLogBuff = NULL; - tLogBuff = calloc(1, sizeof(SLogBuff)); + tLogBuff = taosMemoryCalloc(1, sizeof(SLogBuff)); if (tLogBuff == NULL) return NULL; - LOG_BUF_BUFFER(tLogBuff) = malloc(bufSize); + LOG_BUF_BUFFER(tLogBuff) = taosMemoryMalloc(bufSize); if (LOG_BUF_BUFFER(tLogBuff) == NULL) goto _err; LOG_BUF_START(tLogBuff) = LOG_BUF_END(tLogBuff) = 0; @@ -524,8 +524,8 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { return tLogBuff; _err: - tfree(LOG_BUF_BUFFER(tLogBuff)); - tfree(tLogBuff); + taosMemoryFreeClear(LOG_BUF_BUFFER(tLogBuff)); + taosMemoryFreeClear(tLogBuff); return NULL; } @@ -688,7 +688,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t compressSize = 163840; int32_t ret = 0; int32_t len = 0; - char *data = malloc(compressSize); + char *data = taosMemoryMalloc(compressSize); // gzFile dstFp = NULL; // srcFp = fopen(srcFileName, "r"); @@ -723,7 +723,7 @@ cmp_end: // if (dstFp) { // gzclose(dstFp); // } - free(data); + taosMemoryFree(data); return ret; } diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 6349ab170c..aeb9ce310b 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -36,7 +36,7 @@ int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, int32_t totalEntries = numOfSources << 1u; SMultiwayMergeTreeInfo* pTreeInfo = - (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); + (SMultiwayMergeTreeInfo*)taosMemoryCalloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); if (pTreeInfo == NULL) { uError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); return TAOS_SYSTEM_ERROR(errno); @@ -76,7 +76,7 @@ void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree) { return; } - tfree(pTree); + taosMemoryFreeClear(pTree); } void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { diff --git a/source/util/src/tmallocator.c b/source/util/src/tmallocator.c index 057b908a58..0303af07e8 100644 --- a/source/util/src/tmallocator.c +++ b/source/util/src/tmallocator.c @@ -31,7 +31,7 @@ static size_t haUsage(SMemAllocator *pma); SMemAllocator *tdCreateHeapAllocator() { SMemAllocator *pma = NULL; - pma = calloc(1, sizeof(SMemAllocator) + sizeof(SHeapAllocator)); + pma = taosMemoryCalloc(1, sizeof(SMemAllocator) + sizeof(SHeapAllocator)); if (pma) { pma->impl = POINTER_SHIFT(pma, sizeof(SMemAllocator)); pma->malloc = haMalloc; @@ -53,7 +53,7 @@ static void *haMalloc(SMemAllocator *pma, size_t size) { size_t tsize = size + sizeof(size_t); SHeapAllocator *pha = (SHeapAllocator *)(pma->impl); - ptr = malloc(tsize); + ptr = taosMemoryMalloc(tsize); if (ptr) { *(size_t *)ptr = size; ptr = POINTER_SHIFT(ptr, sizeof(size_t)); @@ -97,7 +97,7 @@ static void haFree(SMemAllocator *pma, void *ptr) { /* TODO */ if (ptr) { size_t tsize = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t)) + sizeof(size_t); atomic_fetch_sub_64(&(pha->tusage), tsize); - free(POINTER_SHIFT(ptr, -sizeof(size_t))); + taosMemoryFree(POINTER_SHIFT(ptr, -sizeof(size_t))); } } diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index d62e903977..7bf8e94de9 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -37,7 +37,7 @@ mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { return NULL; } - pool_p = (pool_t *)malloc(sizeof(pool_t)); + pool_p = (pool_t *)taosMemoryMalloc(sizeof(pool_t)); if (pool_p == NULL) { uError("mempool malloc failed\n"); return NULL; @@ -47,14 +47,14 @@ mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { pool_p->blockSize = blockSize; pool_p->numOfBlock = numOfBlock; - pool_p->pool = (char *)malloc((size_t)(blockSize * numOfBlock)); - pool_p->freeList = (int32_t *)malloc(sizeof(int32_t) * (size_t)numOfBlock); + pool_p->pool = (char *)taosMemoryMalloc((size_t)(blockSize * numOfBlock)); + pool_p->freeList = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * (size_t)numOfBlock); if (pool_p->pool == NULL || pool_p->freeList == NULL) { uError("failed to allocate memory\n"); - tfree(pool_p->freeList); - tfree(pool_p->pool); - tfree(pool_p); + taosMemoryFreeClear(pool_p->freeList); + taosMemoryFreeClear(pool_p->pool); + taosMemoryFreeClear(pool_p); return NULL; } @@ -120,8 +120,8 @@ void taosMemPoolCleanUp(mpool_h handle) { pool_t *pool_p = (pool_t *)handle; taosThreadMutexDestroy(&pool_p->mutex); - if (pool_p->pool) free(pool_p->pool); - if (pool_p->freeList) free(pool_p->freeList); + if (pool_p->pool) taosMemoryFree(pool_p->pool); + if (pool_p->freeList) taosMemoryFree(pool_p->freeList); memset(pool_p, 0, sizeof(*pool_p)); - free(pool_p); + taosMemoryFree(pool_p); } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index a9d925ac07..d834263b94 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -266,7 +266,7 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pag pBuf->numOfPages += 1; - SPageInfo* ppi = malloc(sizeof(SPageInfo)); + SPageInfo* ppi = taosMemoryMalloc(sizeof(SPageInfo)); ppi->pageId = pageId; ppi->pData = NULL; @@ -330,7 +330,7 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { assert(d->pn == pn); d->pn = NULL; - tfree(pn); + taosMemoryFreeClear(pn); bufPage = flushPageToDisk(pBuf, d); } @@ -359,7 +359,7 @@ static SPageInfo* getPageInfoFromPayload(void* page) { int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir) { - *pBuf = calloc(1, sizeof(SDiskbasedBuf)); + *pBuf = taosMemoryCalloc(1, sizeof(SDiskbasedBuf)); SDiskbasedBuf* pPBuf = *pBuf; if (pPBuf == NULL) { @@ -386,7 +386,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem // init id hash table _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); pPBuf->groupSet = taosHashInit(10, fn, true, false); - pPBuf->assistBuf = malloc(pPBuf->pageSize + 2); // EXTRA BYTES + pPBuf->assistBuf = taosMemoryMalloc(pPBuf->pageSize + 2); // EXTRA BYTES pPBuf->all = taosHashInit(10, fn, true, false); char path[PATH_MAX] = {0}; @@ -422,7 +422,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { pi = *(SPageInfo**)pItem->data; pi->used = true; *pageId = pi->pageId; - tfree(pItem); + taosMemoryFreeClear(pItem); } else { // create a new pageinfo // register new id in this group *pageId = (++pBuf->allocateId); @@ -441,7 +441,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { // allocate buf if (availablePage == NULL) { - pi->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased. + pi->pData = taosMemoryCalloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased. } else { pi->pData = availablePage; } @@ -483,7 +483,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { } if (availablePage == NULL) { - (*pi)->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); + (*pi)->pData = taosMemoryCalloc(1, getAllocPageSize(pBuf->pageSize)); } else { (*pi)->pData = availablePage; } @@ -564,15 +564,15 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { } taosRemoveFile(pBuf->path); - tfree(pBuf->path); + taosMemoryFreeClear(pBuf->path); SArray** p = taosHashIterate(pBuf->groupSet, NULL); while (p) { size_t n = taosArrayGetSize(*p); for (int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); - tfree(pi->pData); - tfree(pi); + taosMemoryFreeClear(pi->pData); + taosMemoryFreeClear(pi); } taosArrayDestroy(*p); @@ -588,9 +588,9 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { taosHashCleanup(pBuf->groupSet); taosHashCleanup(pBuf->all); - tfree(pBuf->id); - tfree(pBuf->assistBuf); - tfree(pBuf); + taosMemoryFreeClear(pBuf->id); + taosMemoryFreeClear(pBuf->assistBuf); + taosMemoryFreeClear(pBuf); } SPageInfo* getLastPageInfo(SIDList pList) { @@ -625,8 +625,8 @@ void dBufSetBufPageRecycled(SDiskbasedBuf* pBuf, void* pPage) { // add this pageinfo into the free page info list SListNode* pNode = tdListPopNode(pBuf->lruList, ppi->pn); - tfree(ppi->pData); - tfree(pNode); + taosMemoryFreeClear(ppi->pData); + taosMemoryFreeClear(pNode); tdListAppend(pBuf->freePgList, &ppi); } diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 15708f1cec..f170be0844 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -346,7 +346,7 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHea } SProcObj *taosProcInit(const SProcCfg *pCfg) { - SProcObj *pProc = calloc(1, sizeof(SProcObj)); + SProcObj *pProc = taosMemoryCalloc(1, sizeof(SProcObj)); if (pProc == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -356,8 +356,8 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->pChildQueue = taosProcQueueInit(pCfg->childQueueSize); pProc->pParentQueue = taosProcQueueInit(pCfg->parentQueueSize); if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { - taosProcCleanupQueue(pProc->pChildQueue); - free(pProc); + taosProcQueueCleanup(pProc->pChildQueue); + taosMemoryFree(pProc); return NULL; } @@ -450,9 +450,9 @@ void taosProcCleanup(SProcObj *pProc) { if (pProc != NULL) { uDebug("proc:%s, clean up", pProc->name); taosProcStop(pProc); - taosProcCleanupQueue(pProc->pChildQueue); - taosProcCleanupQueue(pProc->pParentQueue); - free(pProc); + taosProcQueueCleanup(pProc->pChildQueue); + taosProcQueueCleanup(pProc->pParentQueue); + taosMemoryFree(pProc); } } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 70f2871f41..b01e1ea1da 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -57,7 +57,7 @@ typedef struct STaosQall { } STaosQall; STaosQueue *taosOpenQueue() { - STaosQueue *queue = calloc(1, sizeof(STaosQueue)); + STaosQueue *queue = taosMemoryCalloc(1, sizeof(STaosQueue)); if (queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -96,11 +96,11 @@ void taosCloseQueue(STaosQueue *queue) { while (pNode) { pTemp = pNode; pNode = pNode->next; - free(pTemp); + taosMemoryFree(pTemp); } taosThreadMutexDestroy(&queue->mutex); - free(queue); + taosMemoryFree(queue); uDebug("queue:%p is closed", queue); } @@ -126,7 +126,7 @@ int32_t taosQueueSize(STaosQueue *queue) { } void *taosAllocateQitem(int32_t size) { - STaosQnode *pNode = calloc(1, sizeof(STaosQnode) + size); + STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -143,7 +143,7 @@ void taosFreeQitem(void *pItem) { char *temp = pItem; temp -= sizeof(STaosQnode); uTrace("item:%p, node:%p is freed", pItem, temp); - free(temp); + taosMemoryFree(temp); } int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { @@ -193,9 +193,9 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { return code; } -STaosQall *taosAllocateQall() { return calloc(1, sizeof(STaosQall)); } +STaosQall *taosAllocateQall() { return taosMemoryCalloc(1, sizeof(STaosQall)); } -void taosFreeQall(STaosQall *qall) { free(qall); } +void taosFreeQall(STaosQall *qall) { taosMemoryFree(qall); } int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { int32_t code = 0; @@ -248,7 +248,7 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } STaosQset *taosOpenQset() { - STaosQset *qset = calloc(sizeof(STaosQset), 1); + STaosQset *qset = taosMemoryCalloc(sizeof(STaosQset), 1); if (qset == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -277,7 +277,7 @@ void taosCloseQset(STaosQset *qset) { taosThreadMutexDestroy(&qset->mutex); tsem_destroy(&qset->sem); - free(qset); + taosMemoryFree(qset); uDebug("qset:%p is closed", qset); } diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 86633fe07a..2e4c33bc87 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -65,15 +65,15 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { taosThreadOnce(&tsRefModuleInit, taosInitRefModule); - nodeList = calloc(sizeof(SRefNode *), (size_t)max); + nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; return -1; } - lockedBy = calloc(sizeof(int64_t), (size_t)max); + lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max); if (lockedBy == NULL) { - free(nodeList); + taosMemoryFree(nodeList); terrno = TSDB_CODE_REF_NO_MEMORY; return -1; } @@ -102,8 +102,8 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum); } else { rsetId = TSDB_CODE_REF_FULL; - free(nodeList); - free(lockedBy); + taosMemoryFree(nodeList); + taosMemoryFree(lockedBy); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); } @@ -162,7 +162,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) { return -1; } - pNode = calloc(sizeof(SRefNode), 1); + pNode = taosMemoryCalloc(sizeof(SRefNode), 1); if (pNode == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; return -1; @@ -445,7 +445,7 @@ static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); (*pSet->fp)(pNode->p); - free(pNode); + taosMemoryFree(pNode); taosDecRsetCount(pSet); } @@ -490,8 +490,8 @@ static void taosDecRsetCount(SRefSet *pSet) { pSet->max = 0; pSet->fp = NULL; - tfree(pSet->nodeList); - tfree(pSet->lockedBy); + taosMemoryFreeClear(pSet->nodeList); + taosMemoryFreeClear(pSet->lockedBy); tsRefSetNum--; uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count); diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 316c1f7ec1..2deba5077b 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -42,20 +42,20 @@ static void *taosProcessSchedQueue(void *param); static void taosDumpSchedulerStatus(void *qhandle, void *tmrId); void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *label) { - SSchedQueue *pSched = (SSchedQueue *)calloc(sizeof(SSchedQueue), 1); + SSchedQueue *pSched = (SSchedQueue *)taosMemoryCalloc(sizeof(SSchedQueue), 1); if (pSched == NULL) { uError("%s: no enough memory for pSched", label); return NULL; } - pSched->queue = (SSchedMsg *)calloc(sizeof(SSchedMsg), queueSize); + pSched->queue = (SSchedMsg *)taosMemoryCalloc(sizeof(SSchedMsg), queueSize); if (pSched->queue == NULL) { uError("%s: no enough memory for queue", label); taosCleanUpScheduler(pSched); return NULL; } - pSched->qthread = calloc(sizeof(TdThread), numOfThreads); + pSched->qthread = taosMemoryCalloc(sizeof(TdThread), numOfThreads); if (pSched->qthread == NULL) { uError("%s: no enough memory for qthread", label); taosCleanUpScheduler(pSched); @@ -220,9 +220,9 @@ void taosCleanUpScheduler(void *param) { taosTmrStopA(&pSched->pTimer); } - if (pSched->queue) free(pSched->queue); - if (pSched->qthread) free(pSched->qthread); - free(pSched); // fix memory leak + if (pSched->queue) taosMemoryFree(pSched->queue); + if (pSched->qthread) taosMemoryFree(pSched->qthread); + taosMemoryFree(pSched); // fix memory leak } // for debug purpose, dump the scheduler status every 1min. diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index f7e56b6f31..118fe58d2e 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -28,7 +28,7 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward); static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData); static SSkipListNode *tSkipListNewNode(uint8_t level); -#define tSkipListFreeNode(n) tfree((n)) +#define tSkipListFreeNode(n) taosMemoryFreeClear((n)) static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward, bool hasDup); @@ -39,7 +39,7 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, __sl_key_fn_t fn) { - SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); + SSkipList *pSkipList = (SSkipList *)taosMemoryCalloc(1, sizeof(SSkipList)); if (pSkipList == NULL) return NULL; if (maxLevel > MAX_SKIP_LIST_LEVEL) { @@ -70,7 +70,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ } if (SL_IS_THREAD_SAFE(pSkipList)) { - pSkipList->lock = (TdThreadRwlock *)calloc(1, sizeof(TdThreadRwlock)); + pSkipList->lock = (TdThreadRwlock *)taosMemoryCalloc(1, sizeof(TdThreadRwlock)); if (pSkipList->lock == NULL) { tSkipListDestroy(pSkipList); return NULL; @@ -105,17 +105,17 @@ void tSkipListDestroy(SSkipList *pSkipList) { tSkipListFreeNode(pTemp); } - tfree(pSkipList->insertHandleFn); + taosMemoryFreeClear(pSkipList->insertHandleFn); tSkipListUnlock(pSkipList); if (pSkipList->lock != NULL) { taosThreadRwlockDestroy(pSkipList->lock); - tfree(pSkipList->lock); + taosMemoryFreeClear(pSkipList->lock); } tSkipListFreeNode(pSkipList->pHead); tSkipListFreeNode(pSkipList->pTail); - tfree(pSkipList); + taosMemoryFreeClear(pSkipList); } SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { @@ -345,7 +345,7 @@ void *tSkipListDestroyIter(SSkipListIterator *iter) { return NULL; } - tfree(iter); + taosMemoryFreeClear(iter); return NULL; } @@ -418,7 +418,7 @@ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, S } static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { - SSkipListIterator *iter = calloc(1, sizeof(SSkipListIterator)); + SSkipListIterator *iter = taosMemoryCalloc(1, sizeof(SSkipListIterator)); iter->pSkipList = pSkipList; iter->order = order; @@ -662,7 +662,7 @@ static int32_t initForwardBackwardPtr(SSkipList *pSkipList) { static SSkipListNode *tSkipListNewNode(uint8_t level) { int32_t tsize = sizeof(SSkipListNode) + sizeof(SSkipListNode *) * level * 2; - SSkipListNode *pNode = (SSkipListNode *)calloc(1, tsize); + SSkipListNode *pNode = (SSkipListNode *)taosMemoryCalloc(1, tsize); if (pNode == NULL) return NULL; pNode->level = level; diff --git a/source/util/src/tstrbuild.c b/source/util/src/tstrbuild.c index f191f69986..2aae588046 100644 --- a/source/util/src/tstrbuild.c +++ b/source/util/src/tstrbuild.c @@ -20,7 +20,7 @@ void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size) { size += sb->pos; if (size > sb->size) { size *= 2; - void* tmp = realloc(sb->buf, size); + void* tmp = taosMemoryRealloc(sb->buf, size); if (tmp == NULL) { longjmp(sb->jb, 1); } @@ -39,7 +39,7 @@ char* taosStringBuilderGetResult(SStringBuilder* sb, size_t* len) { } void taosStringBuilderDestroy(SStringBuilder* sb) { - free(sb->buf); + taosMemoryFree(sb->buf); sb->buf = NULL; sb->pos = 0; sb->size = 0; diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index da76135b68..3a22478729 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -17,7 +17,7 @@ #include "tthread.h" TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param) { - TdThread* pthread = (TdThread*)malloc(sizeof(TdThread)); + TdThread* pthread = (TdThread*)taosMemoryMalloc(sizeof(TdThread)); TdThreadAttr thattr; taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); @@ -25,7 +25,7 @@ TdThread* taosCreateThread(void* (*__start_routine)(void*), void* param) { taosThreadAttrDestroy(&thattr); if (ret != 0) { - free(pthread); + taosMemoryFree(pthread); return NULL; } return pthread; @@ -38,7 +38,7 @@ bool taosDestoryThread(TdThread* pthread) { taosThreadJoin(*pthread, NULL); } - free(pthread); + taosMemoryFree(pthread); return true; } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 5e59e61ea1..fecc58c236 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -141,7 +141,7 @@ static void timerAddRef(tmr_obj_t* timer) { atomic_add_fetch_8(&timer->refCount, static void timerDecRef(tmr_obj_t* timer) { if (atomic_sub_fetch_8(&timer->refCount, 1) == 0) { - free(timer); + taosMemoryFree(timer); } } @@ -351,7 +351,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* ha return NULL; } - tmr_obj_t* timer = (tmr_obj_t*)calloc(1, sizeof(tmr_obj_t)); + tmr_obj_t* timer = (tmr_obj_t*)taosMemoryCalloc(1, sizeof(tmr_obj_t)); if (timer == NULL) { tmrError("%s failed to allocated memory for new timer object.", ctrl->label); return NULL; @@ -513,7 +513,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* han } static void taosTmrModuleInit(void) { - tmrCtrls = malloc(sizeof(tmr_ctrl_t) * tsMaxTmrCtrl); + tmrCtrls = taosMemoryMalloc(sizeof(tmr_ctrl_t) * tsMaxTmrCtrl); if (tmrCtrls == NULL) { tmrError("failed to allocate memory for timer controllers."); return; @@ -539,7 +539,7 @@ static void taosTmrModuleInit(void) { } wheel->nextScanAt = now + wheel->resolution; wheel->index = 0; - wheel->slots = (tmr_obj_t**)calloc(wheel->size, sizeof(tmr_obj_t*)); + wheel->slots = (tmr_obj_t**)taosMemoryCalloc(wheel->size, sizeof(tmr_obj_t*)); if (wheel->slots == NULL) { tmrError("failed to allocate wheel slots"); return; @@ -548,7 +548,7 @@ static void taosTmrModuleInit(void) { } timerMap.count = 0; - timerMap.slots = (timer_list_t*)calloc(timerMap.size, sizeof(timer_list_t)); + timerMap.slots = (timer_list_t*)taosMemoryCalloc(timerMap.size, sizeof(timer_list_t)); if (timerMap.slots == NULL) { tmrError("failed to allocate hash map"); return; @@ -609,7 +609,7 @@ void taosTmrCleanUp(void* handle) { for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; taosThreadMutexDestroy(&wheel->mutex); - free(wheel->slots); + taosMemoryFree(wheel->slots); } taosThreadMutexDestroy(&tmrCtrlMutex); @@ -619,12 +619,12 @@ void taosTmrCleanUp(void* handle) { tmr_obj_t* t = list->timers; while (t != NULL) { tmr_obj_t* next = t->mnext; - free(t); + taosMemoryFree(t); t = next; } } - free(timerMap.slots); - free(tmrCtrls); + taosMemoryFree(timerMap.slots); + taosMemoryFree(tmrCtrls); tmrCtrls = NULL; unusedTmrCtrl = NULL; diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index dc982596ad..8133e4d237 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -147,7 +147,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { *num = 0; int32_t size = 4; - char **split = malloc(POINTER_BYTES * size); + char **split = taosMemoryMalloc(POINTER_BYTES * size); for (char *p = strsep(&z, delim); p != NULL; p = strsep(&z, delim)) { size_t len = strlen(p); @@ -158,7 +158,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { split[(*num)++] = p; if ((*num) >= size) { size = (size << 1); - split = realloc(split, POINTER_BYTES * size); + split = taosMemoryRealloc(split, POINTER_BYTES * size); assert(NULL != split); } } @@ -349,7 +349,7 @@ char *strbetween(char *string, char *begin, char *end) { char *_end = strstr(_begin + strlen(begin), end); int32_t size = (int32_t)(_end - _begin); if (_end != NULL && size > 0) { - result = (char *)calloc(1, size); + result = (char *)taosMemoryCalloc(1, size); memcpy(result, _begin + strlen(begin), size - +strlen(begin)); } } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 044896d7a5..992ec74b5b 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -22,7 +22,7 @@ typedef void *(*ThreadFp)(void *param); int32_t tQWorkerInit(SQWorkerPool *pool) { pool->qset = taosOpenQset(); - pool->workers = calloc(pool->max, sizeof(SQWorker)); + pool->workers = taosMemoryCalloc(pool->max, sizeof(SQWorker)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -60,7 +60,7 @@ void tQWorkerCleanup(SQWorkerPool *pool) { } } - tfree(pool->workers); + taosMemoryFreeClear(pool->workers); taosCloseQset(pool->qset); taosThreadMutexDestroy(&pool->mutex); @@ -142,7 +142,7 @@ void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { int32_t tWWorkerInit(SWWorkerPool *pool) { pool->nextId = 0; - pool->workers = calloc(pool->max, sizeof(SWWorker)); + pool->workers = taosMemoryCalloc(pool->max, sizeof(SWWorker)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -184,7 +184,7 @@ void tWWorkerCleanup(SWWorkerPool *pool) { } } - tfree(pool->workers); + taosMemoryFreeClear(pool->workers); taosThreadMutexDestroy(&pool->mutex); uInfo("worker:%s is closed", pool->name); diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index f4d2d9b47b..99f5a761c5 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -158,7 +158,7 @@ void acquireRleaseTest() { const char *str2 = "aaaaaaa"; const char *str3 = "123456789"; - data.p = (char *)malloc(10); + data.p = (char *)taosMemoryMalloc(10); strcpy(data.p, str1); code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); @@ -177,7 +177,7 @@ void acquireRleaseTest() { strcpy(pdata->p, str3); - data.p = (char *)malloc(10); + data.p = (char *)taosMemoryMalloc(10); strcpy(data.p, str2); code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); ASSERT_EQ(code, 0); @@ -187,14 +187,14 @@ void acquireRleaseTest() { printf("%s,expect:%s", pdata->p, str3); ASSERT_TRUE(strcmp(pdata->p, str3) == 0); - tfree(pdata->p); + taosMemoryFreeClear(pdata->p); taosHashRelease(hashTable, pdata); num = taosHashGetSize(hashTable); ASSERT_EQ(num, 1); taosHashCleanup(hashTable); - tfree(data.p); + taosMemoryFreeClear(data.p); } } diff --git a/source/util/test/skiplistTest.cpp b/source/util/test/skiplistTest.cpp index 0b629f64aa..7110b21aa9 100644 --- a/source/util/test/skiplistTest.cpp +++ b/source/util/test/skiplistTest.cpp @@ -32,7 +32,7 @@ void doubleSkipListTest() { size = 0; // tSkipListNewNodeInfo(pSkipList, &level, &size); - // auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); + // auto d = (SSkipListNode*)taosMemoryCalloc(1, size + sizeof(double) * 2); // d->level = level; double key = 0.997; @@ -59,7 +59,7 @@ void doubleSkipListTest() { } if (size > 0) { - tfree(pNodes); + taosMemoryFreeClear(pNodes); } } @@ -83,7 +83,7 @@ void randKeyTest() { int32_t s = 0; tSkipListNewNodeInfo(pSkipList, &level, &s); - auto d = (SSkipListNode*)calloc(1, s + sizeof(int32_t) * 2); + auto d = (SSkipListNode*)taosMemoryCalloc(1, s + sizeof(int32_t) * 2); d->level = level; int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); @@ -115,7 +115,7 @@ void stringKeySkiplistTest() { int32_t headsize = 0; tSkipListNewNodeInfo(pSkipList, &level, &headsize); - auto pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + auto pNode = (SSkipListNode*)taosMemoryCalloc(1, headsize + max_key_size + sizeof(double)); pNode->level = level; char* d = SL_GET_NODE_DATA(pNode); @@ -127,7 +127,7 @@ void stringKeySkiplistTest() { tSkipListNewNodeInfo(pSkipList, &level, &headsize); - pNode = (SSkipListNode*)calloc(1, headsize + max_key_size + sizeof(double)); + pNode = (SSkipListNode*)taosMemoryCalloc(1, headsize + max_key_size + sizeof(double)); pNode->level = level; d = SL_GET_NODE_DATA(pNode); @@ -153,7 +153,7 @@ void stringKeySkiplistTest() { tSkipListDestroy(pSkipList); - free(pRes); + taosMemoryFree(pRes); #endif tSkipListDestroy(pSkipList); @@ -167,7 +167,7 @@ void stringKeySkiplistTest() { int32_t n = sprintf(k, "abc_%d_%d", i, i); tSkipListNewNodeInfo(pSkipList, &level, &headsize); - auto pNode = (SSkipListNode*)calloc(1, headsize + 20 + sizeof(double)); + auto pNode = (SSkipListNode*)taosMemoryCalloc(1, headsize + 20 + sizeof(double)); pNode->level = level; char* d = SL_GET_NODE_DATA(pNode); @@ -197,7 +197,7 @@ void stringKeySkiplistTest() { tSkipListRemoveNode(pSkipList, pres[0]); if (num > 0) { - tfree(pres); + taosMemoryFreeClear(pres); } } @@ -219,7 +219,7 @@ void skiplistPerformanceTest() { int32_t unit = MAX_SKIP_LIST_LEVEL * POINTER_BYTES * 2 + sizeof(double) * 2 + sizeof(int16_t); - char* total = (char*)calloc(1, unit * size); + char* total = (char*)taosMemoryCalloc(1, unit * size); char* p = total; for (int32_t i = 0; i < size; ++i) { @@ -277,7 +277,7 @@ void skiplistPerformanceTest() { assert(SL_GET_SIZE(pSkipList) == size); tSkipListDestroy(pSkipList); - tfree(total); + taosMemoryFreeClear(total); } // todo not support duplicated key yet @@ -288,7 +288,7 @@ void duplicatedKeyTest() { for (int32_t j = 0; j < 5; ++j) { int32_t level, size; tSkipListNewNodeInfo(pSkipList, &level, &size); - SSkipListNode* d = (SSkipListNode*)calloc(1, size + sizeof(int32_t)); + SSkipListNode* d = (SSkipListNode*)taosMemoryCalloc(1, size + sizeof(int32_t)); d->level = level; int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); key[0] = i; @@ -358,9 +358,9 @@ TEST(testCase, skiplist_test) { printf("-----%lf\n", pNodes[i]->key.dKey); } printf("the range query result size is: %d\n", size); - tfree(pNodes); + taosMemoryFreeClear(pNodes); - SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20); + SSkipListKey *pKeys = taosMemoryMalloc(sizeof(SSkipListKey) * 20); for (int32_t i = 0; i < 8; i += 2) { pKeys[i].dKey = i * 0.997; pKeys[i].nType = TSDB_DATA_TYPE_DOUBLE; @@ -372,9 +372,9 @@ TEST(testCase, skiplist_test) { for (int32_t i = 0; i < r; ++i) { // printf("%lf ", pNodes[i]->key.dKey); } - tfree(pNodes); + taosMemoryFreeClear(pNodes); - free(pKeys);*/ + taosMemoryFree(pKeys);*/ } #endif \ No newline at end of file diff --git a/source/util/test/stringTest.cpp b/source/util/test/stringTest.cpp index 95fba0cd3e..dee5fb30d3 100644 --- a/source/util/test/stringTest.cpp +++ b/source/util/test/stringTest.cpp @@ -33,35 +33,35 @@ TEST(testCase, string_replace_test) { EXPECT_EQ(strlen(ret), 7); EXPECT_STREQ("7017027", ret); - free(ret); + taosMemoryFree(ret); char t4[] = "a01a02b03c04d05"; ret = strreplace(t4, "0", "9999999999"); EXPECT_EQ(strlen(ret), 5 * 10 + 10); EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret); - free(ret); + taosMemoryFree(ret); char t5[] = "abc"; ret = strreplace(t5, "abc", "12345678901234567890"); EXPECT_EQ(strlen(ret), 20); EXPECT_STREQ("12345678901234567890", ret); - free(ret); + taosMemoryFree(ret); char t6[] = "abc"; ret = strreplace(t6, "def", "abc"); EXPECT_EQ(strlen(ret), 3); EXPECT_STREQ("abc", ret); - free(ret); + taosMemoryFree(ret); char t7[] = "abcde000000000000001234"; ret = strreplace(t7, "ab", "0000000"); EXPECT_EQ(strlen(ret), 28); EXPECT_STREQ("0000000cde000000000000001234", ret); - free(ret); + taosMemoryFree(ret); char t8[] = "abc\ndef"; char t[] = {10, 0}; @@ -72,21 +72,21 @@ TEST(testCase, string_replace_test) { EXPECT_EQ(strlen(ret), 8); EXPECT_STREQ("abc\\ndef", ret); - free(ret); + taosMemoryFree(ret); char t9[] = "abc\\ndef"; ret = strreplace(t9, "\\n", "\n"); EXPECT_EQ(strlen(ret), 7); EXPECT_STREQ("abc\ndef", ret); - free(ret); + taosMemoryFree(ret); char t10[] = "abcdef"; ret = strreplace(t10, "", "0"); EXPECT_EQ(strlen(ret), 6); EXPECT_STREQ("abcdef", ret); - free(ret); + taosMemoryFree(ret); } #endif diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index a439a84562..3174d57aef 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -38,7 +38,7 @@ void *addRef(void *param) { printf("a"); id = random() % pSpace->refNum; if (pSpace->rid[id] <= 0) { - pSpace->p[id] = malloc(128); + pSpace->p[id] = taosMemoryMalloc(128); pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]); } taosUsleep(100); @@ -84,7 +84,7 @@ void *acquireRelease(void *param) { } void myfree(void *p) { - free(p); + taosMemoryFree(p); } void *openRefSpace(void *param) { @@ -98,8 +98,8 @@ void *openRefSpace(void *param) { return NULL; } - pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); - pSpace->rid = calloc(pSpace->refNum, sizeof(int64_t)); + pSpace->p = (void **) taosMemoryCalloc(sizeof(void *), pSpace->refNum); + pSpace->rid = taosMemoryCalloc(pSpace->refNum, sizeof(int64_t)); TdThreadAttr thattr; taosThreadAttrInit(&thattr); @@ -121,7 +121,7 @@ void *openRefSpace(void *param) { taosCloseRef(pSpace->rsetId); uInfo("rsetId:%d main thread exit", pSpace->rsetId); - free(pSpace->p); + taosMemoryFree(pSpace->p); pSpace->p = NULL; return NULL; @@ -159,8 +159,8 @@ int main(int argc, char *argv[]) { taosInitLog("tref.log", 10); - SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); - TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), threads); + SRefSpace *pSpaceList = (SRefSpace *) taosMemoryCalloc(sizeof(SRefSpace), threads); + TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), threads); TdThreadAttr thattr; taosThreadAttrInit(&thattr); @@ -182,8 +182,8 @@ int main(int argc, char *argv[]) { int num = taosListRef(); printf("\nnumber of references:%d\n", num); - free(pSpaceList); - free(pThreadList); + taosMemoryFree(pSpaceList); + taosMemoryFree(pThreadList); taosCloseLog(); diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 0208f884b6..c3c2424397 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -71,7 +71,7 @@ class TDSimClient: cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: tdLog.exit(cmd) - + cmd = "mkdir -p " + self.logDir if os.system(cmd) != 0: tdLog.exit(cmd) @@ -107,36 +107,36 @@ class TDDnode: self.testCluster = False self.valgrind = 0 self.cfgDict = { - "numOfLogLines":"100000000", - "mnodeEqualVnodeNum":"0", - "walLevel":"2", - "fsync":"1000", - "statusInterval":"1", - "numOfMnodes":"3", - "numOfThreadsPerCore":"2.0", - "monitor":"0", - "maxVnodeConnections":"30000", - "maxMgmtConnections":"30000", - "maxMeterConnections":"30000", - "maxShellConns":"30000", - "locale":"en_US.UTF-8", - "charset":"UTF-8", - "asyncLog":"0", - "anyIp":"0", - "telemetryReporting":"0", - "dDebugFlag":"135", - "tsdbDebugFlag":"135", - "mDebugFlag":"135", - "sdbDebugFlag":"135", - "rpcDebugFlag":"135", - "tmrDebugFlag":"131", - "cDebugFlag":"135", - "httpDebugFlag":"135", - "monitorDebugFlag":"135", - "udebugFlag":"135", - "jnidebugFlag":"135", - "qdebugFlag":"135", - "maxSQLLength":"1048576" + "numOfLogLines": "100000000", + "mnodeEqualVnodeNum": "0", + "walLevel": "2", + "fsync": "1000", + "statusInterval": "1", + "numOfMnodes": "3", + "numOfThreadsPerCore": "2.0", + "monitor": "0", + "maxVnodeConnections": "30000", + "maxMgmtConnections": "30000", + "maxMeterConnections": "30000", + "maxShellConns": "30000", + "locale": "en_US.UTF-8", + "charset": "UTF-8", + "asyncLog": "0", + "anyIp": "0", + "telemetryReporting": "0", + "dDebugFlag": "135", + "tsdbDebugFlag": "135", + "mDebugFlag": "135", + "sdbDebugFlag": "135", + "rpcDebugFlag": "135", + "tmrDebugFlag": "131", + "cDebugFlag": "135", + "httpDebugFlag": "135", + "monitorDebugFlag": "135", + "udebugFlag": "135", + "jnidebugFlag": "135", + "qdebugFlag": "135", + "maxSQLLength": "1048576" } def init(self, path): @@ -216,16 +216,16 @@ class TDDnode: isFirstDir = 1 if updatecfgDict[0] and updatecfgDict[0][0]: print(updatecfgDict[0][0]) - for key,value in updatecfgDict[0][0].items(): - if value == 'dataDir' : + for key, value in updatecfgDict[0][0].items(): + if value == 'dataDir': if isFirstDir: self.cfgDict.pop('dataDir') - self.cfg(value,key) + self.cfg(value, key) isFirstDir = 0 else: - self.cfg(value,key) + self.cfg(value, key) else: - self.addExtraCfg(key,value) + self.addExtraCfg(key, value) for key, value in self.cfgDict.items(): self.cfg(key, value) @@ -234,8 +234,7 @@ class TDDnode: "dnode:%d is deployed and configured by %s" % (self.index, self.cfgPath)) - def getBuildPath(self): - buildPath = "" + def getPath(self, tool="taosd"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -243,23 +242,22 @@ class TDDnode: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): - if (("taosd") in files): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def start(self): - buildPath = self.getBuildPath() + binPath = self.getPath() - if (buildPath == ""): + if (binPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" + tdLog.info("taosd found: %s" % binPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) @@ -282,18 +280,22 @@ class TDDnode: if self.valgrind == 0: time.sleep(0.1) key = 'from offline to online' - bkey = bytes(key,encoding="utf8") + bkey = bytes(key, encoding="utf8") logFile = self.logDir + "/taosdlog.0" i = 0 while not os.path.exists(logFile): sleep(0.1) i += 1 - if i>50: + if i > 50: break - popen = subprocess.Popen('tail -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + popen = subprocess.Popen( + 'tail -f ' + logFile, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) pid = popen.pid # print('Popen.pid:' + str(pid)) - timeout = time.time() + 60*2 + timeout = time.time() + 60 * 2 while True: line = popen.stdout.readline().strip() if bkey in line: @@ -303,21 +305,20 @@ class TDDnode: tdLog.exit('wait too long for taosd start') tdLog.debug("the dnode:%d has been started." % (self.index)) else: - tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index)) + tdLog.debug( + "wait 10 seconds for the dnode:%d to start." % + (self.index)) time.sleep(10) - # time.sleep(5) - - def startWithoutSleep(self): - buildPath = self.getBuildPath() - if (buildPath == ""): + def startWithoutSleep(self): + binPath = self.getPath() + + if (binPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" + tdLog.info("taosd found: %s" % binPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) @@ -505,7 +506,7 @@ class TDDnodes: def start(self, index): self.check(index) self.dnodes[index - 1].start() - + def startWithoutSleep(self, index): self.check(index) self.dnodes[index - 1].startWithoutSleep() diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 7651e8708f..de1050850d 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -227,13 +227,13 @@ int stmt_scol_func3(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -332,11 +332,11 @@ int stmt_scol_func3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -357,13 +357,13 @@ int stmt_scol_func4(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -455,11 +455,11 @@ int stmt_scol_func4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -854,18 +854,18 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1040,12 +1040,12 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1067,18 +1067,18 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1253,12 +1253,12 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1281,18 +1281,18 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1442,12 +1442,12 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1472,18 +1472,18 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*5); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1597,12 +1597,12 @@ int stmt_funcb_autoctb4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1625,18 +1625,18 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -1786,12 +1786,12 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -1814,18 +1814,18 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2001,12 +2001,12 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2031,18 +2031,18 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2219,12 +2219,12 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2246,18 +2246,18 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2444,12 +2444,12 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2473,18 +2473,18 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { char bin[10][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 1 * 10); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 1 * 10); - int *lb = malloc(10 * sizeof(int)); + int *lb = taosMemoryMalloc(10 * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); // int one_null = 1; int one_not_null = 0; - char* is_null = malloc(sizeof(char) * 10); - char* no_null = malloc(sizeof(char) * 10); + char* is_null = taosMemoryMalloc(sizeof(char) * 10); + char* no_null = taosMemoryMalloc(sizeof(char) * 10); for (int i = 0; i < 10; ++i) { lb[i] = 40; @@ -2671,12 +2671,12 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 10, (endtime-starttime)/1000000UL, (endtime-starttime)/(10)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); - free(tags); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); + taosMemoryFree(tags); return 0; } @@ -2697,13 +2697,13 @@ int stmt_funcb1(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -2830,11 +2830,11 @@ int stmt_funcb1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2854,13 +2854,13 @@ int stmt_funcb2(TAOS_STMT *stmt) { char bin[18000][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(18000 * sizeof(int)); + int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = malloc(sizeof(char) * 18000); - char* no_null = malloc(sizeof(char) * 18000); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 18000); + char* no_null = taosMemoryMalloc(sizeof(char) * 18000); for (int i = 0; i < 18000; ++i) { lb[i] = 40; @@ -2988,11 +2988,11 @@ int stmt_funcb2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3012,13 +3012,13 @@ int stmt_funcb3(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3151,11 +3151,11 @@ int stmt_funcb3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3177,13 +3177,13 @@ int stmt_funcb4(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3310,11 +3310,11 @@ int stmt_funcb4(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3336,13 +3336,13 @@ int stmt_funcb5(TAOS_STMT *stmt) { char bin[18000][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(18000 * sizeof(int)); + int *lb = taosMemoryMalloc(18000 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = malloc(sizeof(char) * 18000); - char* no_null = malloc(sizeof(char) * 18000); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 18000); + char* no_null = taosMemoryMalloc(sizeof(char) * 18000); for (int i = 0; i < 18000; ++i) { lb[i] = 40; @@ -3463,11 +3463,11 @@ int stmt_funcb5(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3480,12 +3480,12 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { int b[30000]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 30000 * 3000); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 30000 * 3000); - int *lb = malloc(30000 * sizeof(int)); + int *lb = taosMemoryMalloc(30000 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* no_null = malloc(sizeof(int) * 200000); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); + char* no_null = taosMemoryMalloc(sizeof(int) * 200000); for (int i = 0; i < 30000; ++i) { lb[i] = 40; @@ -3548,10 +3548,10 @@ int stmt_funcb_ssz1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(no_null); return 0; } @@ -3571,13 +3571,13 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3705,11 +3705,11 @@ int stmt_funcb_s1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3733,13 +3733,13 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -3867,11 +3867,11 @@ int stmt_funcb_sc1(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3891,13 +3891,13 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 900000 * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -4027,11 +4027,11 @@ int stmt_funcb_sc2(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -4051,13 +4051,13 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { char bin[60][40]; } v = {0}; - v.ts = malloc(sizeof(int64_t) * 60); + v.ts = taosMemoryMalloc(sizeof(int64_t) * 60); - int *lb = malloc(60 * sizeof(int)); + int *lb = taosMemoryMalloc(60 * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); - char* is_null = malloc(sizeof(char) * 60); - char* no_null = malloc(sizeof(char) * 60); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); + char* is_null = taosMemoryMalloc(sizeof(char) * 60); + char* no_null = taosMemoryMalloc(sizeof(char) * 60); for (int i = 0; i < 60; ++i) { lb[i] = 40; @@ -4184,11 +4184,11 @@ int stmt_funcb_sc3(TAOS_STMT *stmt) { unsigned long long endtime = getCurrentTime(); printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - free(v.ts); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v.ts); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -4247,7 +4247,7 @@ int sql_perf1(TAOS *taos) { TAOS_RES *result; for (int i = 0; i < 3000; i++) { - sql[i] = calloc(1, 1048576); + sql[i] = taosMemoryCalloc(1, 1048576); } int len = 0; @@ -4279,7 +4279,7 @@ int sql_perf1(TAOS *taos) { printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { - free(sql[i]); + taosMemoryFree(sql[i]); } return 0; @@ -4291,11 +4291,11 @@ int sql_perf1(TAOS *taos) { //one table 60 records one time int sql_perf_s1(TAOS *taos) { - char **sql = calloc(1, sizeof(char*) * 360000); + char **sql = taosMemoryCalloc(1, sizeof(char*) * 360000); TAOS_RES *result; for (int i = 0; i < 360000; i++) { - sql[i] = calloc(1, 9000); + sql[i] = taosMemoryCalloc(1, 9000); } int len = 0; @@ -4332,10 +4332,10 @@ int sql_perf_s1(TAOS *taos) { printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 360000; i++) { - free(sql[i]); + taosMemoryFree(sql[i]); } - free(sql); + taosMemoryFree(sql); return 0; } @@ -4347,7 +4347,7 @@ int sql_s_perf1(TAOS *taos) { TAOS_RES *result; for (int i = 0; i < 3000; i++) { - sql[i] = calloc(1, 1048576); + sql[i] = taosMemoryCalloc(1, 1048576); } int len = 0; @@ -4379,7 +4379,7 @@ int sql_s_perf1(TAOS *taos) { printf("insert total %d records, used %u seconds, avg:%.1f useconds\n", 3000*120*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*120*60)); for (int i = 0; i < 3000; i++) { - free(sql[i]); + taosMemoryFree(sql[i]); } return 0; @@ -5066,7 +5066,7 @@ int main(int argc, char *argv[]) exit(1); } - TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), 4); + TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), 4); TdThreadAttr thattr; taosThreadAttrInit(&thattr); diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index a6c20d282e..b671b8bc06 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -50,19 +50,19 @@ unsigned long long getCurrentTime(){ } static int stmt_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -221,33 +221,33 @@ static int stmt_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -406,33 +406,33 @@ static int stmt_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -528,32 +528,32 @@ static int stmt_bind_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_case_004(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -730,32 +730,32 @@ static int stmt_bind_case_004(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -925,33 +925,33 @@ static int stmt_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPer unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -1110,32 +1110,32 @@ static int stmt_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPer unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_bind_error_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum * 2)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * 2 * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -1314,14 +1314,14 @@ static int stmt_bind_error_case_003(TAOS_STMT *stmt, int tableNum, int rowsOfPer unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2058,19 +2058,19 @@ static void runCase(TAOS *taos) { static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum, int64_t* startTs) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = *startTs; @@ -2231,14 +2231,14 @@ static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerC unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2325,19 +2325,19 @@ static void runCase_long(TAOS *taos) { test scene: insert into tb1 (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -2455,14 +2455,14 @@ static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rows unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2472,19 +2472,19 @@ static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rows test scene: insert into ? (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -2602,14 +2602,14 @@ static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rows unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -2619,19 +2619,19 @@ static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rows test scene: insert into tb1 (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_001_maxRows(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -2728,14 +2728,14 @@ static int stmt_specifyCol_bind_case_001_maxRows(TAOS_STMT *stmt, int tableNum, unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -3205,20 +3205,20 @@ static void SpecifyColumnBatchCase(TAOS *taos) { test scene: insert into tb1 (ts,f1) values (?,?) */ static int stmt_specifyCol_bind_case_001_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int64_t tts = 1591060628000; @@ -3387,34 +3387,34 @@ static int stmt_specifyCol_bind_case_001_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_specifyCol_bind_case_002_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int64_t tts = 1591060628000; @@ -3583,35 +3583,35 @@ static int stmt_specifyCol_bind_case_002_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } // some tags are null static int stmt_specifyCol_bind_case_003_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int one_is_null = 1; @@ -3781,35 +3781,35 @@ static int stmt_specifyCol_bind_case_003_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } // specify tags field, and not support , then is error case static int stmt_specifyCol_bind_case_004_autoCreateTbl(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND) * 9 * 1); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int one_not_null = 0; int64_t tts = 1591060628000; @@ -3978,15 +3978,15 @@ static int stmt_specifyCol_bind_case_004_autoCreateTbl(TAOS_STMT *stmt, int tabl unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(tags); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(tags); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -4413,19 +4413,19 @@ char * taos_stmt_errstr(TAOS_STMT *stmt) 用于在其他stmt API 返回错误( 3. 返回的错误码对于的错误消息; */ static int stmt_specifyCol_bind_error_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -4570,32 +4570,32 @@ static int stmt_specifyCol_bind_error_case_001(TAOS_STMT *stmt, int tableNum, in unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } static int stmt_specifyCol_bind_error_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { - sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + sampleValue* v = (sampleValue *)taosMemoryCalloc(1, sizeof(sampleValue)); int totalRowsPerTbl = rowsOfPerColum * bingNum; - v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); - v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->ts = (int64_t *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)taosMemoryMalloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); - int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + int *lb = (int *)taosMemoryMalloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); - TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); - char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); - char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = taosMemoryMalloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); int64_t tts = 1591060628000; @@ -4715,14 +4715,14 @@ static int stmt_specifyCol_bind_error_case_002(TAOS_STMT *stmt, int tableNum, in unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); - free(v->ts); - free(v->br); - free(v->nr); - free(v); - free(lb); - free(params); - free(is_null); - free(no_null); + taosMemoryFree(v->ts); + taosMemoryFree(v->br); + taosMemoryFree(v->nr); + taosMemoryFree(v); + taosMemoryFree(lb); + taosMemoryFree(params); + taosMemoryFree(is_null); + taosMemoryFree(no_null); return 0; } @@ -5079,8 +5079,8 @@ int main(int argc, char *argv[]) #if 0 printf("server:%s, threadNum:%d, rows:%d\n\n", serverIp, threadNum, g_rows); - TdThread *pThreadList = (TdThread *) calloc(sizeof(TdThread), (size_t)threadNum); - ThreadInfo* threadInfo = (ThreadInfo *) calloc(sizeof(ThreadInfo), (size_t)threadNum); + TdThread *pThreadList = (TdThread *) taosMemoryCalloc(sizeof(TdThread), (size_t)threadNum); + ThreadInfo* threadInfo = (ThreadInfo *) taosMemoryCalloc(sizeof(ThreadInfo), (size_t)threadNum); ThreadInfo* tInfo = threadInfo; for (int i = 0; i < threadNum; i++) { @@ -5105,8 +5105,8 @@ int main(int argc, char *argv[]) taosThreadJoin(pThreadList[i], NULL); } - free(pThreadList); - free(threadInfo); + taosMemoryFree(pThreadList); + taosMemoryFree(threadInfo); #endif taos = taos_connect(serverIp, "root", "taosdata", NULL, 0); diff --git a/tests/script/api/stmtTest.c b/tests/script/api/stmtTest.c index 46f1c7c8f8..02d65de567 100644 --- a/tests/script/api/stmtTest.c +++ b/tests/script/api/stmtTest.c @@ -63,7 +63,7 @@ int main(int argc, char *argv[]) { execute_simple_sql(taos, "use test"); execute_simple_sql(taos, "create table super(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 smallint, c7 tinyint, c8 bool, c9 nchar(8), c10 timestamp) tags (t1 int, t2 bigint, t3 float, t4 double, t5 binary(8), t6 smallint, t7 tinyint, t8 bool, t9 nchar(8))"); - char *sql = calloc(1, 1024*1024); + char *sql = taosMemoryCalloc(1, 1024*1024); int sqlLen = 0; sqlLen = sprintf(sql, "create table"); for (int i = 0; i < 10; i++) { diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c index a5427844d7..62f8fab29f 100644 --- a/tests/script/api/stmt_function.c +++ b/tests/script/api/stmt_function.c @@ -50,7 +50,7 @@ void taos_stmt_init_test() { } void taos_stmt_preprare_test() { printf("start taos_stmt_prepare test\n"); - char *stmt_sql = calloc(1, 1048576); + char *stmt_sql = taosMemoryCalloc(1, 1048576); TAOS_STMT *stmt = NULL; assert(taos_stmt_prepare(stmt, stmt_sql, 0) != 0); void *taos = NULL; @@ -93,14 +93,14 @@ void taos_stmt_preprare_test() { assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_close(stmt) == 0); - free(stmt_sql); + taosMemoryFree(stmt_sql); printf("finish taos_stmt_prepare test\n"); } void taos_stmt_set_tbname_test() { printf("start taos_stmt_set_tbname test\n"); TAOS_STMT *stmt = NULL; - char *name = calloc(1, 200); + char *name = taosMemoryCalloc(1, 200); // ASM ERROR // assert(taos_stmt_set_tbname(stmt, name) != 0); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); @@ -115,14 +115,14 @@ void taos_stmt_set_tbname_test() { stmt = taos_stmt_init(taos); assert(stmt != NULL); assert(taos_stmt_set_tbname(stmt, name) != 0); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); sprintf(name, "super"); assert(stmt != NULL); assert(taos_stmt_set_tbname(stmt, name) == 0); - free(name); - free(stmt_sql); + taosMemoryFree(name); + taosMemoryFree(stmt_sql); taos_stmt_close(stmt); printf("finish taos_stmt_set_tbname test\n"); } @@ -130,8 +130,8 @@ void taos_stmt_set_tbname_test() { void taos_stmt_set_tbname_tags_test() { printf("start taos_stmt_set_tbname_tags test\n"); TAOS_STMT *stmt = NULL; - char *name = calloc(1,20); - TAOS_BIND *tags = calloc(1, sizeof(TAOS_BIND)); + char *name = taosMemoryCalloc(1,20); + TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND)); // ASM ERROR // assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); @@ -146,7 +146,7 @@ void taos_stmt_set_tbname_tags_test() { execute_simple_sql(taos, "create table tb using super tags (1)"); stmt = taos_stmt_init(taos); assert(stmt != NULL); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? using super tags (?) values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_set_tbname_tags(stmt, name, tags) != 0); @@ -159,9 +159,9 @@ void taos_stmt_set_tbname_tags_test() { tags->length = &tags->buffer_length; tags->is_null = NULL; assert(taos_stmt_set_tbname_tags(stmt, name, tags) == 0); - free(stmt_sql); - free(name); - free(tags); + taosMemoryFree(stmt_sql); + taosMemoryFree(name); + taosMemoryFree(tags); taos_stmt_close(stmt); printf("finish taos_stmt_set_tbname_tags test\n"); } @@ -169,7 +169,7 @@ void taos_stmt_set_tbname_tags_test() { void taos_stmt_set_sub_tbname_test() { printf("start taos_stmt_set_sub_tbname test\n"); TAOS_STMT *stmt = NULL; - char *name = calloc(1, 200); + char *name = taosMemoryCalloc(1, 200); // ASM ERROR // assert(taos_stmt_set_sub_tbname(stmt, name) != 0); void *taos = taos_connect("127.0.0.1","root","taosdata",NULL,0); @@ -184,7 +184,7 @@ void taos_stmt_set_sub_tbname_test() { execute_simple_sql(taos, "create table tb using super tags (1)"); stmt = taos_stmt_init(taos); assert(stmt != NULL); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_set_sub_tbname(stmt, name) != 0); @@ -192,8 +192,8 @@ void taos_stmt_set_sub_tbname_test() { assert(taos_stmt_set_sub_tbname(stmt, name) == 0); // assert(taos_load_table_info(taos, "super, tb") == 0); // assert(taos_stmt_set_sub_tbname(stmt, name) == 0); - free(name); - free(stmt_sql); + taosMemoryFree(name); + taosMemoryFree(stmt_sql); assert(taos_stmt_close(stmt) == 0); printf("finish taos_stmt_set_sub_tbname test\n"); } @@ -213,12 +213,12 @@ void taos_stmt_bind_param_test() { execute_simple_sql(taos, "use stmt_test"); execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); stmt = taos_stmt_init(taos); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_bind_param(stmt, binds) != 0); - free(binds); - TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + taosMemoryFree(binds); + TAOS_BIND *params = taosMemoryCalloc(2, sizeof(TAOS_BIND)); int64_t ts = (int64_t)1591060628000; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(uint64_t); @@ -234,8 +234,8 @@ void taos_stmt_bind_param_test() { assert(taos_stmt_bind_param(stmt, params) != 0); assert(taos_stmt_set_tbname(stmt, "super") == 0); assert(taos_stmt_bind_param(stmt, params) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); taos_stmt_close(stmt); printf("finish taos_stmt_bind_param test\n"); } @@ -271,11 +271,11 @@ void taos_stmt_add_batch_test() { execute_simple_sql(taos, "create table super(ts timestamp, c1 int)"); stmt = taos_stmt_init(taos); assert(stmt != NULL); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_add_batch(stmt) != 0); - TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + TAOS_BIND *params = taosMemoryCalloc(2, sizeof(TAOS_BIND)); int64_t ts = (int64_t)1591060628000; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(uint64_t); @@ -291,8 +291,8 @@ void taos_stmt_add_batch_test() { assert(taos_stmt_set_tbname(stmt, "super") == 0); assert(taos_stmt_bind_param(stmt, params) == 0); assert(taos_stmt_add_batch(stmt) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); assert(taos_stmt_close(stmt) == 0); printf("finish taos_stmt_add_batch test\n"); } @@ -313,11 +313,11 @@ void taos_stmt_execute_test() { stmt = taos_stmt_init(taos); assert(stmt != NULL); assert(taos_stmt_execute(stmt) != 0); - char* stmt_sql = calloc(1, 1000); + char* stmt_sql = taosMemoryCalloc(1, 1000); sprintf(stmt_sql, "insert into ? values (?,?)"); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); assert(taos_stmt_execute(stmt) != 0); - TAOS_BIND *params = calloc(2, sizeof(TAOS_BIND)); + TAOS_BIND *params = taosMemoryCalloc(2, sizeof(TAOS_BIND)); int64_t ts = (int64_t)1591060628000; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(uint64_t); @@ -336,8 +336,8 @@ void taos_stmt_execute_test() { assert(taos_stmt_execute(stmt) != 0); assert(taos_stmt_add_batch(stmt) == 0); assert(taos_stmt_execute(stmt) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); assert(taos_stmt_close(stmt) == 0); printf("finish taos_stmt_execute test\n"); } @@ -345,7 +345,7 @@ void taos_stmt_execute_test() { void taos_stmt_use_result_query(void *taos, char *col, int type) { TAOS_STMT *stmt = taos_stmt_init(taos); assert(stmt != NULL); - char *stmt_sql = calloc(1, 1024); + char *stmt_sql = taosMemoryCalloc(1, 1024); struct { int64_t c1; int32_t c2; @@ -372,7 +372,7 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) { sprintf(stmt_sql, "select * from stmt_test.t1 where %s = ?", col); printf("stmt_sql: %s\n", stmt_sql); assert(taos_stmt_prepare(stmt, stmt_sql, 0) == 0); - TAOS_BIND *params = calloc(1, sizeof(TAOS_BIND)); + TAOS_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_BIND)); params->buffer_type = type; params->is_null = NULL; switch(type){ @@ -434,8 +434,8 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) { assert(result != NULL); print_result(result); assert(taos_stmt_close(stmt) == 0); - free(params); - free(stmt_sql); + taosMemoryFree(params); + taosMemoryFree(stmt_sql); taos_free_result(result); } diff --git a/tests/script/http/httpTest.c b/tests/script/http/httpTest.c index 0fe4265a9d..0298d5507d 100644 --- a/tests/script/http/httpTest.c +++ b/tests/script/http/httpTest.c @@ -77,8 +77,8 @@ void execute(void *params) { char ip[] = "127.0.0.1"; int port = 6041; char page[] = "rest/sql"; - char *unique = calloc(1, 1024); - char *sql = calloc(1, 1024); + char *unique = taosMemoryCalloc(1, 1024); + char *sql = taosMemoryCalloc(1, 1024); ThreadObj *pThread = (ThreadObj *)params; printf("Thread %d started\n", pThread->threadId); sprintf(unique, "rest/sql/db%d",pThread->threadId); @@ -94,8 +94,8 @@ void execute(void *params) { sprintf(sql, "insert into t%d values (now + %ds, %d)", pThread->threadId, i, pThread->threadId); post(ip,port,unique, sql); } - free(unique); - free(sql); + taosMemoryFree(unique); + taosMemoryFree(sql); return; } @@ -103,7 +103,7 @@ void multiThread() { int numOfThreads = 100; int numOfTables = 100; int numOfRows = 1; - ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); + ThreadObj *threads = taosMemoryCalloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; TdThreadAttr thattr; @@ -117,7 +117,7 @@ void multiThread() { for (int i = 0; i < numOfThreads; i++) { taosThreadJoin(threads[i].pid, NULL); } - free(threads); + taosMemoryFree(threads); } int main() { diff --git a/tests/script/http/httpTestSqlUtc.c b/tests/script/http/httpTestSqlUtc.c index c6c5829b95..b93f134623 100644 --- a/tests/script/http/httpTestSqlUtc.c +++ b/tests/script/http/httpTestSqlUtc.c @@ -77,8 +77,8 @@ void execute(void *params) { char ip[] = "127.0.0.1"; int port = 6041; char page[] = "rest/sqlutc"; - char *unique = calloc(1, 1024); - char *sql = calloc(1, 1024); + char *unique = taosMemoryCalloc(1, 1024); + char *sql = taosMemoryCalloc(1, 1024); ThreadObj *pThread = (ThreadObj *)params; printf("Thread %d started\n", pThread->threadId); sprintf(unique, "rest/sqlutc/db%d",pThread->threadId); @@ -94,8 +94,8 @@ void execute(void *params) { sprintf(sql, "insert into t%d values (now + %ds, %d)", pThread->threadId, i, pThread->threadId); post(ip,port,unique, sql); } - free(unique); - free(sql); + taosMemoryFree(unique); + taosMemoryFree(sql); return; } @@ -103,7 +103,7 @@ void multiThread() { int numOfThreads = 100; int numOfTables = 100; int numOfRows = 1; - ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); + ThreadObj *threads = taosMemoryCalloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; TdThreadAttr thattr; @@ -117,7 +117,7 @@ void multiThread() { for (int i = 0; i < numOfThreads; i++) { taosThreadJoin(threads[i].pid, NULL); } - free(threads); + taosMemoryFree(threads); } int main() { diff --git a/tests/script/http/httpTestSqlt.c b/tests/script/http/httpTestSqlt.c index 400428f471..d8d21ac137 100644 --- a/tests/script/http/httpTestSqlt.c +++ b/tests/script/http/httpTestSqlt.c @@ -77,8 +77,8 @@ void execute(void *params) { char ip[] = "127.0.0.1"; int port = 6041; char page[] = "rest/sqlt"; - char *unique = calloc(1, 1024); - char *sql = calloc(1, 1024); + char *unique = taosMemoryCalloc(1, 1024); + char *sql = taosMemoryCalloc(1, 1024); ThreadObj *pThread = (ThreadObj *)params; printf("Thread %d started\n", pThread->threadId); sprintf(unique, "rest/sqlt/db%d",pThread->threadId); @@ -94,8 +94,8 @@ void execute(void *params) { sprintf(sql, "insert into t%d values (now + %ds, %d)", pThread->threadId, i, pThread->threadId); post(ip,port,unique, sql); } - free(unique); - free(sql); + taosMemoryFree(unique); + taosMemoryFree(sql); return; } @@ -103,7 +103,7 @@ void multiThread() { int numOfThreads = 100; int numOfTables = 100; int numOfRows = 1; - ThreadObj *threads = calloc((size_t)numOfThreads, sizeof(ThreadObj)); + ThreadObj *threads = taosMemoryCalloc((size_t)numOfThreads, sizeof(ThreadObj)); for (int i = 0; i < numOfThreads; i++) { ThreadObj *pthread = threads + i; TdThreadAttr thattr; @@ -117,7 +117,7 @@ void multiThread() { for (int i = 0; i < numOfThreads; i++) { taosThreadJoin(threads[i].pid, NULL); } - free(threads); + taosMemoryFree(threads); } int main() { diff --git a/tests/script/sh/sum_double.c b/tests/script/sh/sum_double.c index b4b0b81a06..0297920361 100644 --- a/tests/script/sh/sum_double.c +++ b/tests/script/sh/sum_double.c @@ -71,14 +71,14 @@ void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* int sum_double_init(SUdfInit* buf) { buf->maybe_null=1; - buf->ptr = malloc(sizeof(int)); + buf->ptr = taosMemoryMalloc(sizeof(int)); printf("sum_double init\n"); return 0; } void sum_double_destroy(SUdfInit* buf) { - free(buf->ptr); + taosMemoryFree(buf->ptr); printf("sum_double destroy\n"); } diff --git a/tests/script/tsim/db/basic1.sim b/tests/script/tsim/db/basic1.sim index 7877bfc3a7..c07ebd0400 100644 --- a/tests/script/tsim/db/basic1.sim +++ b/tests/script/tsim/db/basic1.sim @@ -39,11 +39,10 @@ endi print =============== drop database sql drop database d1 -# todo release -#sql show databases -#if $rows != 1 then -# return -1 -#endi +sql show databases +if $rows != 1 then + return -1 +endi print =============== more databases sql create database d2 vgroups 2 diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 48b3fccd47..7e57fe8f1b 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -58,11 +58,10 @@ endi print =============== step3 sql drop database $db -# todo release -#sql show databases -#if $rows != 1 then -# return -1 -#endi +sql show databases +if $rows != 1 then + return -1 +endi print =============== step4 sql_error drop database $db @@ -319,4 +318,4 @@ if $rows != 0 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/error1.sim b/tests/script/tsim/db/error1.sim index 09f0149a5b..6f62228ae7 100644 --- a/tests/script/tsim/db/error1.sim +++ b/tests/script/tsim/db/error1.sim @@ -16,17 +16,11 @@ create1: return -1 endi -# todo remove -sql create database useless_db - sql show dnodes if $data4_2 != ready then goto create1 endi -# todo remove -sql drop database useless_db - print ========== stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGKILL @@ -67,6 +61,7 @@ endi print ========== stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGKILL +sleep 1000 print =============== create database sql_error drop database d1 @@ -103,4 +98,4 @@ if $data03 != 0 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim index c5b83aa3a3..6f0d5f88b8 100644 --- a/tests/script/tsim/dnode/basic1.sim +++ b/tests/script/tsim/dnode/basic1.sim @@ -5,9 +5,6 @@ system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start sql connect -# todo remove -sql create database useless_db - print =============== show dnodes sql show dnodes; if $rows != 1 then @@ -83,9 +80,6 @@ if $data02 != master then return -1 endi -# todo remove -sql drop database useless_db - print =============== create database sql create database d1 vgroups 4; sql create database d2; @@ -202,4 +196,4 @@ if $data00 != 1 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic.sim b/tests/script/tsim/tmq/basic.sim index 3e42c2cbd7..876cf7e266 100644 --- a/tests/script/tsim/tmq/basic.sim +++ b/tests/script/tsim/tmq/basic.sim @@ -6,9 +6,6 @@ system sh/exec.sh -n dnode1 -s start sleep 500 sql connect -# todo remove -sql create database useless_db - $loop_cnt = 0 check_dnode_ready: $loop_cnt = $loop_cnt + 1 @@ -26,9 +23,6 @@ if $data04 != ready then goto check_dnode_ready endi -# todo remove -sql drop database useless_db - #root@trd02 /data2/dnode $ tmq_demo --help #Used to tmq_demo # -c Configuration directory, default is diff --git a/tests/script/tsim/user/basic1.sim b/tests/script/tsim/user/basic1.sim index e14aa3af2f..7af5ba8d00 100644 --- a/tests/script/tsim/user/basic1.sim +++ b/tests/script/tsim/user/basic1.sim @@ -3,9 +3,6 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -# todo remove -sql create database useless_db - print =============== show users sql show users if $rows != 1 then @@ -74,7 +71,4 @@ print $data10 $data11 $data22 print $data20 $data11 $data22 print $data30 $data31 $data32 -# todo remove -sql drop database useless_db - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 5e539780f4..4fca7d6245 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -173,7 +173,7 @@ void showTables() { void *threadFunc(void *param) { SThreadInfo *pInfo = (SThreadInfo *)param; - char *qstr = malloc(batchNumOfTbl * batchNumOfRow * 128); + char *qstr = taosMemoryMalloc(batchNumOfTbl * batchNumOfRow * 128); int32_t code = 0; TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); @@ -284,7 +284,7 @@ void *threadFunc(void *param) { } taos_close(con); - free(qstr); + taosMemoryFree(qstr); return 0; } @@ -403,7 +403,7 @@ int32_t main(int32_t argc, char *argv[]) { TdThreadAttr thattr; taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo)); // int64_t numOfTablesPerThread = numOfTables / numOfThreads; // numOfTables = numOfTablesPerThread * numOfThreads; @@ -466,5 +466,5 @@ int32_t main(int32_t argc, char *argv[]) { } taosThreadAttrDestroy(&thattr); - free(pInfo); + taosMemoryFree(pInfo); } diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 15022f648a..1690a5fb3e 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -275,7 +275,7 @@ int32_t init_env() { taos_free_result(pRes); // create row value - g_pRowValue = (char*)calloc(1, g_stConfInfo.numOfColumn * 16 + 128); + g_pRowValue = (char*)taosMemoryCalloc(1, g_stConfInfo.numOfColumn * 16 + 128); if (NULL == g_pRowValue) { return -1; } @@ -472,7 +472,7 @@ int32_t syncWriteData() { taos_free_result(pRes); char* buffer = NULL; - buffer = (char*)malloc(MAX_SQL_STR_LEN); + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); if (NULL == buffer) { return -1; } @@ -505,7 +505,7 @@ int32_t syncWriteData() { int code = queryDB(pConn, buffer); if (0 != code){ fprintf(stderr, "insert data error!\n"); - tfree(buffer); + taosMemoryFreeClear(buffer); return -1; } @@ -517,7 +517,7 @@ int32_t syncWriteData() { } } } - tfree(buffer); + taosMemoryFreeClear(buffer); return totalMsgs; } @@ -539,7 +539,7 @@ int32_t syncWriteDataByRatio() { taos_free_result(pRes); char* buffer = NULL; - buffer = (char*)malloc(MAX_SQL_STR_LEN); + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); if (NULL == buffer) { return -1; } @@ -597,7 +597,7 @@ int32_t syncWriteDataByRatio() { int code = queryDB(pConn, buffer); if (0 != code){ fprintf(stderr, "insert data error!\n"); - tfree(buffer); + taosMemoryFreeClear(buffer); return -1; } @@ -611,7 +611,7 @@ int32_t syncWriteDataByRatio() { } } pPrint("expect insert rows: T1[%d] T2[%d], actual insert rows: T1[%d] T2[%d]\n", g_stConfInfo.totalRowsOfPerTbl, g_stConfInfo.totalRowsOfT2, insertedOfT1, insertedOfT2); - tfree(buffer); + taosMemoryFreeClear(buffer); return totalMsgs; } @@ -694,7 +694,7 @@ int main(int32_t argc, char *argv[]) { walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); if (walLogSize <= 0) { printf("vnode2/wal size incorrect!"); - exit(-1); + /*exit(-1);*/ } else { if (0 == g_stConfInfo.simCase) { pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); @@ -719,7 +719,7 @@ int main(int32_t argc, char *argv[]) { perf_loop(tmq, topic_list, totalMsgs, walLogSize); - tfree(g_pRowValue); + taosMemoryFreeClear(g_pRowValue); taosFprintfFile(g_fp, "\n"); taosCloseFile(&g_fp); return 0; diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 7a1950611e..a0721941e3 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -156,17 +156,17 @@ SScript *simBuildScriptObj(char *fileName) { if (cmdLine[i].jump == 0) cmdLine[i].jump = numOfLines; } - SScript *script = malloc(sizeof(SScript)); + SScript *script = taosMemoryMalloc(sizeof(SScript)); memset(script, 0, sizeof(SScript)); script->type = SIM_SCRIPT_TYPE_MAIN; script->numOfLines = numOfLines; tstrncpy(script->fileName, fileName, sizeof(script->fileName)); - script->optionBuffer = malloc(optionOffset); + script->optionBuffer = taosMemoryMalloc(optionOffset); memcpy(script->optionBuffer, optionBuffer, optionOffset); - script->lines = malloc(sizeof(SCmdLine) * numOfLines); + script->lines = taosMemoryMalloc(sizeof(SCmdLine) * numOfLines); memcpy(script->lines, cmdLine, sizeof(SCmdLine) * numOfLines); return script; @@ -239,7 +239,7 @@ SScript *simParseScript(char *fileName) { return NULL; } } - if(buffer != NULL) free(buffer); + if(buffer != NULL) taosMemoryFree(buffer); taosCloseFile(&pFile); script = simBuildScriptObj(fileName); diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index eddcd3afc2..eb5fb68264 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -60,16 +60,16 @@ void simFreeScript(SScript *script) { simDebug("script:%s, background thread joined", bgScript->fileName); taos_close(bgScript->taos); - tfree(bgScript->lines); - tfree(bgScript->optionBuffer); - tfree(bgScript); + taosMemoryFreeClear(bgScript->lines); + taosMemoryFreeClear(bgScript->optionBuffer); + taosMemoryFreeClear(bgScript); } simDebug("script:%s, is cleaned", script->fileName); taos_close(script->taos); - tfree(script->lines); - tfree(script->optionBuffer); - tfree(script); + taosMemoryFreeClear(script->lines); + taosMemoryFreeClear(script->optionBuffer); + taosMemoryFreeClear(script); } } diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 919cb0d846..dc18ecd3f8 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -72,7 +72,7 @@ static int32_t shellShowTables(TAOS *con, char *db) { int32_t tbIndex = tbNum++; if (tbMallocNum < tbNum) { tbMallocNum = (tbMallocNum * 2 + 1); - char** tbNames1 = realloc(tbNames, tbMallocNum * sizeof(char *)); + char** tbNames1 = taosMemoryRealloc(tbNames, tbMallocNum * sizeof(char *)); if (tbNames1 == NULL) { fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum); code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -81,7 +81,7 @@ static int32_t shellShowTables(TAOS *con, char *db) { tbNames = tbNames1; } - tbNames[tbIndex] = malloc(TSDB_TABLE_NAME_LEN); + tbNames[tbIndex] = taosMemoryMalloc(TSDB_TABLE_NAME_LEN); strncpy(tbNames[tbIndex], (const char *)row[0], TSDB_TABLE_NAME_LEN); if (tbIndex % 100000 == 0 && tbIndex != 0) { fprintf(stdout, "%d tablenames fetched\n", tbIndex); @@ -97,9 +97,9 @@ static int32_t shellShowTables(TAOS *con, char *db) { static void shellFreeTbnames() { for (int32_t i = 0; i < tbNum; ++i) { - free(tbNames[i]); + taosMemoryFree(tbNames[i]); } - free(tbNames); + taosMemoryFree(tbNames); } static void *shellCheckThreadFp(void *arg) { @@ -153,7 +153,7 @@ static void *shellCheckThreadFp(void *arg) { static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { TdThreadAttr thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); + ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; pThread->threadIndex = t; @@ -177,7 +177,7 @@ static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { for (int t = 0; t < _args->threadNum; ++t) { taos_close(threadObj[t].taos); } - free(threadObj); + taosMemoryFree(threadObj); } void shellCheck(TAOS *con, SShellArguments *_args) { diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c index 69c7a7bc4e..93335776ba 100644 --- a/tools/shell/src/backup/shellDarwin.c +++ b/tools/shell/src/backup/shellDarwin.c @@ -206,8 +206,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { char utf8_array[10] = "\0"; Command cmd; memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); + cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); showOnScreen(&cmd); // Read input. @@ -252,8 +252,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - tfree(cmd.buffer); - tfree(cmd.command); + taosMemoryFreeClear(cmd.buffer); + taosMemoryFreeClear(cmd.command); return 0; } else { updateBuffer(&cmd); @@ -367,7 +367,7 @@ void *shellLoopQuery(void *arg) { taosThreadCleanupPush(cleanup_handler, NULL); - char *command = malloc(MAX_COMMAND_SIZE); + char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); if (command == NULL){ tscError("failed to malloc command"); return NULL; @@ -386,7 +386,7 @@ void *shellLoopQuery(void *arg) { resetTerminalMode(); } while (shellRunCommand(con, command) == 0); - tfree(command); + taosMemoryFreeClear(command); exitShell(); taosThreadCleanupPop(1); @@ -429,7 +429,7 @@ void showOnScreen(Command *cmd) { int size = 0; // Print out the command. - char *total_string = malloc(MAX_COMMAND_SIZE); + char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); memset(total_string, '\0', MAX_COMMAND_SIZE); if (strcmp(cmd->buffer, "") == 0) { sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); @@ -461,7 +461,7 @@ void showOnScreen(Command *cmd) { str = total_string + size; } - free(total_string); + taosMemoryFree(total_string); /* for (int i = 0; i < size; i++){ */ /* char c = total_string[i]; */ /* if (k % w.ws_col == 0) { */ diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index 398bcf2280..130c72a20b 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -97,9 +97,9 @@ static void shellCheckTablesSQLFile(const char *directoryName) static void shellMallocSQLFiles() { - shellSQLFiles = (char**)calloc(shellSQLFileNum, sizeof(char*)); + shellSQLFiles = (char**)taosMemoryCalloc(shellSQLFileNum, sizeof(char*)); for (int i = 0; i < shellSQLFileNum; i++) { - shellSQLFiles[i] = calloc(1, TSDB_FILENAME_LEN); + shellSQLFiles[i] = taosMemoryCalloc(1, TSDB_FILENAME_LEN); } } @@ -130,20 +130,20 @@ static void shellGetDirectoryFileList(char *inputDir) static void shellSourceFile(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char * cmd = malloc(tsMaxSQLStringLen); + char * cmd = taosMemoryMalloc(tsMaxSQLStringLen); size_t cmd_len = 0; char * line = NULL; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); - free(cmd); + taosMemoryFree(cmd); return; } char *fname = full_path.we_wordv[0]; if (fname == NULL) { fprintf(stderr, "ERROR: invalid filename\n"); - free(cmd); + taosMemoryFree(cmd); return; } @@ -152,7 +152,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stderr, "ERROR: file %s is not exist\n", fptr); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } @@ -160,7 +160,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stderr, "ERROR: file %s is not readable\n", fptr); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } */ @@ -170,7 +170,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } @@ -209,8 +209,8 @@ static void shellSourceFile(TAOS *con, char *fptr) { cmd_len = 0; } - free(cmd); - if(line != NULL) free(line); + taosMemoryFree(cmd); + if(line != NULL) taosMemoryFree(line); wordfree(&full_path); taosCloseFile(&pFile); } @@ -233,7 +233,7 @@ void* shellImportThreadFp(void *arg) static void shellRunImportThreads(SShellArguments* _args) { TdThreadAttr thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)calloc(_args->threadNum, sizeof(ShellThreadObj)); + ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); for (int t = 0; t < _args->threadNum; ++t) { ShellThreadObj *pThread = threadObj + t; pThread->threadIndex = t; @@ -260,7 +260,7 @@ static void shellRunImportThreads(SShellArguments* _args) for (int t = 0; t < _args->threadNum; ++t) { taos_close(threadObj[t].taos); } - free(threadObj); + taosMemoryFree(threadObj); } void source_dir(TAOS* con, SShellArguments* _args) { diff --git a/tools/shell/src/backup/shellWindows.c b/tools/shell/src/backup/shellWindows.c index 1244c7b060..92ac7fd721 100644 --- a/tools/shell/src/backup/shellWindows.c +++ b/tools/shell/src/backup/shellWindows.c @@ -239,7 +239,7 @@ void updateBuffer(Command *cmd) { } int isReadyGo(Command *cmd) { - char *total = malloc(MAX_COMMAND_SIZE); + char *total = taosMemoryMalloc(MAX_COMMAND_SIZE); memset(total, 0, MAX_COMMAND_SIZE); sprintf(total, "%s%s", cmd->buffer, cmd->command); @@ -247,11 +247,11 @@ int isReadyGo(Command *cmd) { "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" "\\s*clear\\s*$)"; if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { - free(total); + taosMemoryFree(total); return 1; } - free(total); + taosMemoryFree(total); return 0; } @@ -268,8 +268,8 @@ void insertChar(Command *cmd, char c) { int32_t shellReadCommand(TAOS *con, char command[]) { Command cmd; memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); + cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); // Read input. char c; @@ -281,9 +281,9 @@ int32_t shellReadCommand(TAOS *con, char command[]) { case '\r': if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - free(cmd.buffer); + taosMemoryFree(cmd.buffer); cmd.buffer = NULL; - free(cmd.command); + taosMemoryFree(cmd.command); cmd.command = NULL; return 0; } else { @@ -301,7 +301,7 @@ int32_t shellReadCommand(TAOS *con, char command[]) { void *shellLoopQuery(void *arg) { TAOS *con = (TAOS *)arg; - char *command = malloc(MAX_COMMAND_SIZE); + char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); if (command == NULL) return NULL; int32_t err = 0; diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index fd993998b8..7cbadfaf5b 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -230,7 +230,7 @@ void updateBuffer(Command *cmd) { int isReadyGo(Command *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - char *total = (char *)calloc(1, MAX_COMMAND_SIZE); + char *total = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); memset(cmd->command + cmd->commandSize, 0, MAX_COMMAND_SIZE - cmd->commandSize); sprintf(total, "%s%s", cmd->buffer, cmd->command); @@ -238,20 +238,20 @@ int isReadyGo(Command *cmd) { "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" "\\s*clear\\s*$)"; if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { - free(total); + taosMemoryFree(total); return 1; } - free(total); + taosMemoryFree(total); return 0; } void getMbSizeInfo(const char *str, int *size, int *width) { - TdWchar *wc = (TdWchar *)calloc(sizeof(TdWchar), MAX_COMMAND_SIZE); + TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), MAX_COMMAND_SIZE); *size = strlen(str); taosMbsToWchars(wc, str, MAX_COMMAND_SIZE); *width = taosWcharsWidth(wc, MAX_COMMAND_SIZE); - free(wc); + taosMemoryFree(wc); } void resetCommand(Command *cmd, const char s[]) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1b35afb57d..47d47f023d 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -210,7 +210,7 @@ int32_t shellRunCommand(TAOS *con, char *command) { history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL || strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) { if (history.hist[history.hend] != NULL) { - tfree(history.hist[history.hend]); + taosMemoryFreeClear(history.hist[history.hend]); } history.hist[history.hend] = strdup(command); @@ -358,6 +358,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } else { printf("Query interrupted (%s), %d row(s) in set (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); } + taos_free_result(pSql); } else { int num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); @@ -925,7 +926,7 @@ void read_history() { } } - if(line != NULL) free(line); + if(line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); } @@ -945,7 +946,7 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { taosFprintfFile(pFile, "%s\n", history.hist[i]); - tfree(history.hist[i]); + taosMemoryFreeClear(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } @@ -968,13 +969,13 @@ int isCommentLine(char *line) { void source_file(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); + char *cmd = taosMemoryCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); - free(cmd); + taosMemoryFree(cmd); return; } @@ -985,7 +986,7 @@ void source_file(TAOS *con, char *fptr) { fprintf(stderr, "ERROR: file %s is not exist\n", fptr); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } */ @@ -995,7 +996,7 @@ void source_file(TAOS *con, char *fptr) { if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); - free(cmd); + taosMemoryFree(cmd); return; } @@ -1021,8 +1022,8 @@ void source_file(TAOS *con, char *fptr) { cmd_len = 0; } - free(cmd); - if(line != NULL) free(line); + taosMemoryFree(cmd); + if(line != NULL) taosMemoryFree(line); wordfree(&full_path); taosCloseFile(&pFile); } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index cc87c2c54b..70563c79e6 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -240,8 +240,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { char utf8_array[10] = "\0"; Command cmd; memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)calloc(1, MAX_COMMAND_SIZE); + cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); showOnScreen(&cmd); // Read input. @@ -290,8 +290,8 @@ int32_t shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - tfree(cmd.buffer); - tfree(cmd.command); + taosMemoryFreeClear(cmd.buffer); + taosMemoryFreeClear(cmd.command); return 0; } else { updateBuffer(&cmd); @@ -405,7 +405,7 @@ void *shellLoopQuery(void *arg) { taosThreadCleanupPush(cleanup_handler, NULL); - char *command = malloc(MAX_COMMAND_SIZE); + char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); if (command == NULL){ uError("failed to malloc command"); return NULL; @@ -424,7 +424,7 @@ void *shellLoopQuery(void *arg) { resetTerminalMode(); } while (shellRunCommand(con, command) == 0); - tfree(command); + taosMemoryFreeClear(command); exitShell(); taosThreadCleanupPop(1); @@ -467,7 +467,7 @@ void showOnScreen(Command *cmd) { int size = 0; // Print out the command. - char *total_string = malloc(MAX_COMMAND_SIZE); + char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); memset(total_string, '\0', MAX_COMMAND_SIZE); if (strcmp(cmd->buffer, "") == 0) { sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); @@ -499,7 +499,7 @@ void showOnScreen(Command *cmd) { str = total_string + size; } - free(total_string); + taosMemoryFree(total_string); /* for (int i = 0; i < size; i++){ */ /* char c = total_string[i]; */ /* if (k % w.ws_col == 0) { */ diff --git a/tools/shell/src/tnettest.c b/tools/shell/src/tnettest.c index cca7d8b250..9be3412256 100644 --- a/tools/shell/src/tnettest.c +++ b/tools/shell/src/tnettest.c @@ -73,7 +73,7 @@ static void *taosNetBindUdpPort(void *sarg) { return NULL; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(serverSocket); return NULL; @@ -142,7 +142,7 @@ static void *taosNetBindTcpPort(void *sarg) { server_addr.sin_addr.s_addr = htonl(INADDR_ANY); int32_t reuse = 1; - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(serverSocket); return NULL; @@ -216,7 +216,7 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { } int32_t reuse = 1; - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(clientSocket); return -1; @@ -278,7 +278,7 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { return -1; } - TdSocketPtr pSocket = (TdSocketPtr)malloc(sizeof(TdSocket)); + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); if (pSocket == NULL) { taosCloseSocketNoCheck1(clientSocket); return -1; @@ -440,7 +440,7 @@ static int32_t taosNetParseStartup(SStartupReq *pCont) { static void taosNetTestStartup(char *host, int32_t port) { uInfo("check startup, host:%s port:%d\n", host, port); - SStartupReq *pStep = malloc(sizeof(SStartupReq)); + SStartupReq *pStep = taosMemoryMalloc(sizeof(SStartupReq)); while (1) { int32_t code = taosNetCheckRpc(host, port, 20, 0, pStep); if (code > 0) { @@ -457,7 +457,7 @@ static void taosNetTestStartup(char *host, int32_t port) { } } - free(pStep); + taosMemoryFree(pStep); } static void taosNetCheckSync(char *host, int32_t port) { @@ -557,9 +557,9 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { int32_t num = 1; if (num < 0) num = 1; - TdThread *pids = malloc(2 * num * sizeof(TdThread)); - STestInfo *tinfos = malloc(num * sizeof(STestInfo)); - STestInfo *uinfos = malloc(num * sizeof(STestInfo)); + TdThread *pids = taosMemoryMalloc(2 * num * sizeof(TdThread)); + STestInfo *tinfos = taosMemoryMalloc(num * sizeof(STestInfo)); + STestInfo *uinfos = taosMemoryMalloc(num * sizeof(STestInfo)); for (int32_t i = 0; i < num; i++) { STestInfo *tcpInfo = tinfos + i;