diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 6295d8553d..b114f92a41 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -112,6 +112,9 @@ taosBenchmark -f - **-u/--user ** : User name to connect to the TDengine server. Default is root. +- **-U/--supplement-insert ** : + Supplementally insert data without create database and table, optional, default is off. + - **-p/--password ** : The default password to connect to the TDengine server is `taosdata`. @@ -148,6 +151,9 @@ taosBenchmark -f - **-l/--columns ** : specify the number of columns in the super table. If both this parameter and `-b/--data-type` is set, the final result number of columns is the greater of the two. If the number specified by this parameter is greater than the number of columns specified by `-b/--data-type`, the unspecified column type defaults to INT, for example: `-l 5 -b float,double`, then the final column is `FLOAT,DOUBLE,INT,INT,INT`. If the number of columns specified is less than or equal to the number of columns specified by `-b/--data-type`, then the result is the column and type specified by `-b/--data-type`, e.g.: `-l 3 -b float,double,float,bigint`. The last column is `FLOAT,DOUBLE, FLOAT,BIGINT`. +- **-L/--partial-col-num ** : + Specify first numbers of columns has data. Rest of columns' data are NULL. Default is all columns have data. + - **-A/--tag-type ** : The tag column type of the super table. nchar and binary types can both set the length, for example: diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index e067c646b2..afc050298f 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -112,6 +112,9 @@ taosBenchmark -f - **-u/--user ** : 用于连接 TDengine 服务端的用户名,默认为 root 。 +- **-U/--supplement-insert ** : + 写入数据而不提前建数据库和表,默认关闭。 + - **-p/--password ** : 用于连接 TDengine 服务端的密码,默认值为 taosdata。 @@ -148,6 +151,9 @@ taosBenchmark -f - **-l/--columns ** : 超级表的数据列的总数量。如果同时设置了该参数和 `-b/--data-type`,则最后的结果列数为两者取大。如果本参数指定的数量大于 `-b/--data-type` 指定的列数,则未指定的列类型默认为 INT, 例如: `-l 5 -b float,double`, 那么最后的列为 `FLOAT,DOUBLE,INT,INT,INT`。如果 columns 指定的数量小于或等于 `-b/--data-type` 指定的列数,则结果为 `-b/--data-type` 指定的列和类型,例如: `-l 3 -b float,double,float,bigint`,那么最后的列为 `FLOAT,DOUBLE,FLOAT,BIGINT` 。 +- **-L/--partial-col-num **: + 指定某些列写入数据,其他列数据为 NULL。默认所有列都写入数据。 + - **-A/--tag-type ** : 超级表的标签列类型。nchar 和 binary 类型可以同时设置长度,例如: diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index a1fefa6902..c0c3fc7fbc 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -234,8 +234,6 @@ struct STag { // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#define TASSERT(x) ASSERT(x) - #define STR_TO_VARSTR(x, str) \ do { \ VarDataLenT __len = (VarDataLenT)strlen(str); \ diff --git a/include/common/trow.h b/include/common/trow.h index 8b0349c99b..c5293fa45f 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -255,7 +255,7 @@ static FORCE_INLINE void *tdGetBitmapAddrKv(STSRow *pRow, col_id_t nKvCols) { void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols); int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); -bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); +// bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); // ----------------- Tuple row structure(STpRow) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 733270dbed..eace9d7dd6 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -340,12 +340,8 @@ typedef struct tDataTypeDescriptor { } tDataTypeDescriptor; extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX]; - bool isValidDataType(int32_t type); -void setVardataNull(void *val, int32_t type); -//void setNull(void *val, int32_t type, int32_t bytes); -//void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems); void assignVal(char *val, const char *src, int32_t len, int32_t type); void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); void *getDataMin(int32_t type); diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 12e465e0f8..3dee59ab14 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -161,10 +161,6 @@ typedef struct { SMonLogs log; } SMonMmInfo; -int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo); -int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo); -void tFreeSMonMmInfo(SMonMmInfo *pInfo); - typedef struct { SArray *datadirs; // array of SMonDiskDesc } SMonDiskInfo; @@ -176,56 +172,31 @@ typedef struct { SMonLogs log; } SMonVmInfo; -int32_t tSerializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo); -int32_t tDeserializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo); -void tFreeSMonVmInfo(SMonVmInfo *pInfo); - typedef struct { SMonSysInfo sys; SMonLogs log; SQnodeLoad load; } SMonQmInfo; -int32_t tSerializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo); -int32_t tDeserializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo); -void tFreeSMonQmInfo(SMonQmInfo *pInfo); - typedef struct { SMonSysInfo sys; SMonLogs log; } SMonSmInfo; -int32_t tSerializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo); -int32_t tDeserializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo); -void tFreeSMonSmInfo(SMonSmInfo *pInfo); typedef struct { SMonSysInfo sys; SMonLogs log; } SMonBmInfo; -int32_t tSerializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo); -int32_t tDeserializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo); -void tFreeSMonBmInfo(SMonBmInfo *pInfo); - typedef struct { SArray *pVloads; // SVnodeLoad } SMonVloadInfo; -int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo); -int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo); -void tFreeSMonVloadInfo(SMonVloadInfo *pInfo); - typedef struct { int8_t isMnode; SMnodeLoad load; } SMonMloadInfo; -int32_t tSerializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo); -int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo); - -int32_t tSerializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo); -int32_t tDeserializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo); - typedef struct { const char *server; uint16_t port; @@ -245,6 +216,12 @@ void monSetSmInfo(SMonSmInfo *pInfo); void monSetBmInfo(SMonBmInfo *pInfo); void monSendReport(); +void tFreeSMonMmInfo(SMonMmInfo *pInfo); +void tFreeSMonVmInfo(SMonVmInfo *pInfo); +void tFreeSMonQmInfo(SMonQmInfo *pInfo); +void tFreeSMonSmInfo(SMonSmInfo *pInfo); +void tFreeSMonBmInfo(SMonBmInfo *pInfo); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index ec68e8ed71..3d59a69dea 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -68,9 +68,10 @@ int32_t streamStateSessionClear(SStreamState* pState); int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); +int32_t streamStateSessionGetKey(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey); SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key); -SStreamStateCur* streamStateSessionSeekKeyPrev(SStreamState* pState, const SSessionKey* key); +SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key); SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKey* key); int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); diff --git a/include/util/tdef.h b/include/util/tdef.h index bb7f908ad4..936fbdf0d5 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -237,7 +237,6 @@ typedef enum ELogicConditionType { #define TSDB_MAX_BYTES_PER_ROW 49151 #define TSDB_MAX_TAGS_LEN 16384 #define TSDB_MAX_TAGS 128 -#define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_MAX_COL_TAG_NUM (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS) #define TSDB_MAX_JSON_TAG_LEN 16384 @@ -274,8 +273,6 @@ typedef enum ELogicConditionType { #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE #define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value -#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth -#define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MIN_VNODES 16 #define TSDB_MAX_VNODES 512 @@ -285,10 +282,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_REPLICA 5 -#define TSDB_TBNAME_COLUMN_INDEX (-1) -#define TSDB_UD_COLUMN_INDEX (-1000) -#define TSDB_RES_COL_ID (-5000) - +#define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta #define TSDB_MIN_VNODES_PER_DB 1 @@ -398,9 +392,6 @@ typedef enum ELogicConditionType { #define TSDB_MAX_EXPLAIN_RATIO 1 #define TSDB_DEFAULT_EXPLAIN_RATIO 0.001 -#define TSDB_MAX_JOIN_TABLE_NUM 10 -#define TSDB_MAX_UNION_CLAUSE 5 - #define TSDB_DEFAULT_EXPLAIN_VERBOSE false #define TSDB_EXPLAIN_RESULT_ROW_SIZE (16 * 1024) @@ -419,7 +410,6 @@ typedef enum ELogicConditionType { #endif #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode #define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index e279c1fef5..0b7b80649e 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -979,6 +979,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; + SMsgSendInfo* sendInfo = NULL; SCMSubscribeReq req = {0}; int32_t code = -1; @@ -1016,7 +1017,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { void* abuf = buf; tSerializeSCMSubscribeReq(&abuf, &req); - SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) goto FAIL; SMqSubscribeCbParam param = { @@ -1046,6 +1047,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { // avoid double free if msg is sent buf = NULL; + sendInfo = NULL; tsem_wait(¶m.rspSem); tsem_destroy(¶m.rspSem); @@ -1078,8 +1080,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = 0; FAIL: - if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree); + taosArrayDestroyP(req.topicNames, taosMemoryFree); taosMemoryFree(buf); + taosMemoryFree(sendInfo); return code; } @@ -1613,7 +1616,11 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (rspWrapper == NULL) { taosReadAllQitems(tmq->mqueue, tmq->qall); taosGetQitem(tmq->qall, (void**)&rspWrapper); - if (rspWrapper == NULL) return NULL; + + if (rspWrapper == NULL) { + tscDebug("consumer %" PRId64 " mqueue empty", tmq->consumerId); + return NULL; + } } if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { @@ -1711,6 +1718,8 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { void* rspObj; int64_t startTime = taosGetTimestampMs(); + tscDebug("consumer:%" PRId64 ", start poll at %" PRId64, tmq->consumerId, startTime); + #if 0 tmqHandleAllDelayedTask(tmq); tmqPollImpl(tmq, timeout); @@ -1745,15 +1754,18 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { rspObj = tmqHandleAllRsp(tmq, timeout, false); if (rspObj) { + tscDebug("consumer:%" PRId64 ", return rsp", tmq->consumerId); return (TAOS_RES*)rspObj; } else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { + tscDebug("consumer:%" PRId64 ", return null since no committed offset", tmq->consumerId); return NULL; } if (timeout != -1) { int64_t endTime = taosGetTimestampMs(); int64_t leftTime = endTime - startTime; if (leftTime > timeout) { - tscDebug("consumer:%" PRId64 ", (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch); + tscDebug("consumer:%" PRId64 ", (epoch %d) timeout, no rsp, start time %" PRId64 ", end time %" PRId64, + tmq->consumerId, tmq->epoch, startTime, endTime); return NULL; } tsem_timewait(&tmq->rspSem, leftTime * 1000); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a22837d981..fda1a05290 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1858,7 +1858,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memset(pBuf, 0, dataSize); - taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); + (void)taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); printf(" %15s |", pBuf); } break; default: @@ -1946,7 +1946,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memset(pBuf, 0, sizeof(pBuf)); - taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); + (void)taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) return dumpBuf; } break; @@ -2053,7 +2053,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_MEDIUMBLOB: uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); - TASSERT(0); + ASSERT(0); break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { @@ -2084,7 +2084,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB } } else { uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - TASSERT(0); + ASSERT(0); } break; } diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index f1713cfd98..dfaebc99f6 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -21,7 +21,8 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = 0; - strcpy(pEp->fqdn, ep); + memset(pEp->fqdn, 0, TSDB_FQDN_LEN); + strncpy(pEp->fqdn, ep, TSDB_FQDN_LEN - 1); char* temp = strchr(pEp->fqdn, ':'); if (temp) { diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 887c449c56..70682d6cd4 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -90,8 +90,10 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { pName->type = TSDB_TABLE_NAME_T; pName->acctId = acctId; - strcpy(pName->dbname, pDbName); - strcpy(pName->tname, pTableName); + memset(pName->dbname, 0, TSDB_DB_NAME_LEN); + strncpy(pName->dbname, pDbName, TSDB_DB_NAME_LEN - 1); + memset(pName->tname, 0, TSDB_TABLE_NAME_LEN); + strncpy(pName->tname, pTableName, TSDB_TABLE_NAME_LEN - 1); return pName; } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index d5c0f2ddf8..b007075efe 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -481,7 +481,7 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -503,7 +503,7 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa *pValType = ((*pDestByte) & 0x03); break; default: - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -512,7 +512,7 @@ int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pVa int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -546,7 +546,7 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal *pValType = ((*pDestByte) & 0x01); break; default: - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -555,7 +555,7 @@ int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pVal int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -598,7 +598,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { // *pDestByte |= (valType); break; default: - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -607,7 +607,7 @@ int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP - TASSERT(colIdx < tdRowGetNCols(pRow) - 1); + ASSERT(colIdx < tdRowGetNCols(pRow) - 1); if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; return terrno; @@ -621,7 +621,7 @@ int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_ output->val = POINTER_SHIFT(pRow, offset); } #else - TASSERT(0); + ASSERT(0); if (offset < 0) { terrno = TSDB_CODE_INVALID_PARA; output->valType = TD_VTYPE_NONE; @@ -671,7 +671,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp return terrno; } #else - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; #endif @@ -709,7 +709,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -798,7 +798,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou pBuilder->nCols = nCols; pBuilder->nBoundCols = nBoundCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -820,7 +820,7 @@ int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBou int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -831,7 +831,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { TD_ROW_SET_INFO(pBuilder->pBuf, 0); TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); - TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + ASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); uint32_t len = 0; switch (pBuilder->rowType) { @@ -857,7 +857,7 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); break; default: - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -868,12 +868,12 @@ int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { pBuilder->pBuf = (STSRow *)pBuf; if (!pBuilder->pBuf) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } - TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + ASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); uint32_t len = 0; switch (pBuilder->rowType) { @@ -888,7 +888,7 @@ int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { #endif break; default: - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -908,7 +908,7 @@ int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { pBuilder->flen = flen; pBuilder->nCols = nCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -927,7 +927,7 @@ int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, pBuilder->nCols = nCols; pBuilder->nBoundCols = nBoundCols; if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -956,13 +956,13 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT tdGetBitmapValTypeI(pBitmap, colIdx, pValType); break; default: - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; } return TSDB_CODE_SUCCESS; } - +#if 0 bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { TDRowValT valType = 0; tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); @@ -971,10 +971,11 @@ bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) } return false; } +#endif int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1001,7 +1002,7 @@ int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { // *pDestByte |= (valType); break; default: - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1018,7 +1019,7 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int tdSetBitmapValTypeI(pBitmap, colIdx, valType); break; default: - TASSERT(0); + ASSERT(0); terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_FAILED; } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index bb82863e73..e0cc5bd311 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -244,8 +244,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { * 2013-04-12T15:52:01.123+0800 */ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { - int64_t factor = - (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int64_t tzOffset = 0; struct tm tm = {0}; @@ -339,8 +338,8 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) { return true; } -int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePrec, char delim) { - *time = 0; +int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { + *utime = 0; struct tm tm = {0}; char* str; @@ -378,15 +377,12 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePr } } - int64_t factor = - (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); - *time = factor * seconds + fraction; - + *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; return 0; } -int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t timePrec, char delim) { - *time = 0; +int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { + *utime = 0; struct tm tm = {0}; tm.tm_isdst = -1; @@ -411,7 +407,6 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t tim int64_t seconds = taosMktime(&tm); int64_t fraction = 0; - if (*str == '.') { /* parse the second fraction part */ if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) { @@ -419,9 +414,7 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t tim } } - int64_t factor = - (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); - *time = factor * seconds + fraction; + *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; return 0; } @@ -437,58 +430,61 @@ char getPrecisionUnit(int32_t precision) { } } -int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { - assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || +int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPrecision) { + ASSERT(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); - assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || + ASSERT(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || toPrecision == TSDB_TIME_PRECISION_NANO); - double tempResult = (double)time; + + double tempResult = (double)utime; + switch (fromPrecision) { case TSDB_TIME_PRECISION_MILLI: { switch (toPrecision) { case TSDB_TIME_PRECISION_MILLI: - return time; + return utime; case TSDB_TIME_PRECISION_MICRO: tempResult *= 1000; - time *= 1000; + utime *= 1000; goto end_; case TSDB_TIME_PRECISION_NANO: tempResult *= 1000000; - time *= 1000000; + utime *= 1000000; goto end_; } } // end from milli case TSDB_TIME_PRECISION_MICRO: { switch (toPrecision) { case TSDB_TIME_PRECISION_MILLI: - return time / 1000; + return utime / 1000; case TSDB_TIME_PRECISION_MICRO: - return time; + return utime; case TSDB_TIME_PRECISION_NANO: tempResult *= 1000; - time *= 1000; + utime *= 1000; goto end_; } } // end from micro case TSDB_TIME_PRECISION_NANO: { switch (toPrecision) { case TSDB_TIME_PRECISION_MILLI: - return time / 1000000; + return utime / 1000000; case TSDB_TIME_PRECISION_MICRO: - return time / 1000; + return utime / 1000; case TSDB_TIME_PRECISION_NANO: - return time; + return utime; } } // end from nano default: { assert(0); - return time; // only to pass windows compilation + return utime; // only to pass windows compilation } } // end switch fromPrecision + end_: if (tempResult >= (double)INT64_MAX) return INT64_MAX; if (tempResult <= (double)INT64_MIN) return INT64_MIN; // INT64_MIN means NULL - return time; + return utime; } // !!!!notice:there are precision problems, double lose precison if time is too large, for example: diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 3dbd8a05a2..a4e7a12ce4 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "ttypes.h" #include "tcompression.h" -#include "trow.h" const int32_t TYPE_BYTES[16] = { -1, // TSDB_DATA_TYPE_NULL @@ -86,18 +85,6 @@ FORCE_INLINE void *getDataMax(int32_t type) { bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_MAX; } -void setVardataNull(void *val, int32_t type) { - if (type == TSDB_DATA_TYPE_BINARY) { - varDataSetLen(val, sizeof(int8_t)); - *(uint8_t *)varDataVal(val) = TSDB_DATA_BINARY_NULL; - } else if (type == TSDB_DATA_TYPE_NCHAR) { - varDataSetLen(val, sizeof(int32_t)); - *(uint32_t *)varDataVal(val) = TSDB_DATA_NCHAR_NULL; - } else { - assert(0); - } -} - #define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) void assignVal(char *val, const char *src, int32_t len, int32_t type) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 654664269d..2660fa3146 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -113,7 +113,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER) for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { - TASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions); + ASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions); SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i); SDB_SET_INT64(pRaw, dataPos, pRetension->freq, _OVER) SDB_SET_INT64(pRaw, dataPos, pRetension->keep, _OVER) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0e8b3cf949..d16c5b19f3 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -575,7 +575,11 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { tdbTbGet(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pData, &nData); tDecoderInit(&dc, pData, nData); - metaDecodeEntry(&dc, &e); + rc = metaDecodeEntry(&dc, &e); + if (rc < 0) { + tDecoderClear(&dc); + return -1; + } if (type) *type = e.type; @@ -668,6 +672,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl SSchemaWrapper *pSchema; int c; + if (pAlterTbReq->colName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + // search name index ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { @@ -1244,7 +1253,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); tDecoderInit(&dc, pData, nData); - metaDecodeEntry(&dc, &stbEntry); + ret = metaDecodeEntry(&dc, &stbEntry); + if (ret < 0) { + goto end; + } pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 20352bb502..ed491ca182 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -194,7 +194,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } else if ((pr->type & CACHESCAN_RETRIEVE_TYPE_ALL) == CACHESCAN_RETRIEVE_TYPE_ALL) { for (int32_t i = pr->tableIndex; i < numOfTables; ++i) { - STableKeyInfo* pKeyInfo = taosArrayGet(pr->pTableList, i); + STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pr->pTableList, i); code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 460ef611a3..674cdc115f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1054,6 +1054,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { _exit: tsdbFSDestroy(&pCommitter->fs); taosArrayDestroy(pCommitter->aTbDataP); + pCommitter->aTbDataP = NULL; if (code || eno) { tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index c63247c21a..4039520851 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -149,6 +149,7 @@ _exit: void tsdbFSDestroy(STsdbFS *pFS) { if (pFS->pDelFile) { taosMemoryFree(pFS->pDelFile); + pFS->pDelFile = NULL; } for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) { @@ -162,6 +163,7 @@ void tsdbFSDestroy(STsdbFS *pFS) { } taosArrayDestroy(pFS->aDFileSet); + pFS->aDFileSet = NULL; } static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { @@ -296,7 +298,7 @@ static int32_t tsdbLoadFSFromFile(const char *fname, STsdbFS *pFS) { // load binary TdFilePtr pFD = taosOpenFile(fname, TD_FILE_READ); if (pFD == NULL) { - code = terrno; + code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 2f4fdfc5f8..8808f3c50c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -120,44 +120,46 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { return &pInfo->blockData[1]; } - pInfo->currentLoadBlockIndex ^= 1; - if (pIter->pSttBlk != NULL) { // current block not loaded yet - int64_t st = taosGetTimestampUs(); - - SBlockData *pBlock = &pInfo->blockData[pInfo->currentLoadBlockIndex]; - - TABLEID id = {0}; - if (pIter->pSttBlk->suid != 0) { - id.suid = pIter->pSttBlk->suid; - } else { - id.uid = pIter->uid; - } - - code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } - - code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } - - double el = (taosGetTimestampUs() - st) / 1000.0; - pInfo->elapsedTime += el; - pInfo->loadBlocks += 1; - - tsdbDebug("read last block, total load:%d, trigger by uid:%" PRIu64 - ", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s", - pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el, - idStr); - - pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; - tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr); - - pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].nRow : -1; + if (pIter->pSttBlk == NULL) { + return NULL; } + // current block not loaded yet + pInfo->currentLoadBlockIndex ^= 1; + int64_t st = taosGetTimestampUs(); + + SBlockData *pBlock = &pInfo->blockData[pInfo->currentLoadBlockIndex]; + + TABLEID id = {0}; + if (pIter->pSttBlk->suid != 0) { + id.suid = pIter->pSttBlk->suid; + } else { + id.uid = pIter->uid; + } + + code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + double el = (taosGetTimestampUs() - st) / 1000.0; + pInfo->elapsedTime += el; + pInfo->loadBlocks += 1; + + tsdbDebug("read last block, total load:%d, trigger by uid:%" PRIu64 + ", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s", + pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el, + idStr); + + pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; + tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr); + + pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].nRow : -1; return &pInfo->blockData[pInfo->currentLoadBlockIndex]; _exit: @@ -259,7 +261,8 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr) { - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); if (*pIter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -336,7 +339,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1; } + return code; + _exit: + taosMemoryFree(*pIter); return code; } @@ -473,7 +479,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { int32_t iBlockL = pIter->iSttBlk; SBlockData *pBlockData = loadLastBlock(pIter, idStr); - if (pBlockData == NULL && terrno != TSDB_CODE_SUCCESS) { + if (pBlockData == NULL || terrno != TSDB_CODE_SUCCESS) { goto _exit; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ea4e86fb78..11cae00358 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1047,11 +1047,16 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v return pLeftBlock->offset > pRightBlock->offset ? 1 : -1; } -static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { +static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); if (pBlockInfo != NULL) { STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); - int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); + if (pScanInfo == NULL) { + tsdbError("failed to locate the uid:%"PRIu64" in query table uid list, %s", pBlockInfo->uid, idStr); + return TSDB_CODE_INVALID_PARA; + } + + int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetDataBlk); } @@ -1135,7 +1140,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte pBlockIter->index = asc ? 0 : (numOfBlocks - 1); cleanupBlockOrderSupporter(&sup); - doSetCurrentBlock(pBlockIter); + doSetCurrentBlock(pBlockIter, pReader->idStr); return TSDB_CODE_SUCCESS; } @@ -1175,12 +1180,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte taosMemoryFree(pTree); pBlockIter->index = asc ? 0 : (numOfBlocks - 1); - doSetCurrentBlock(pBlockIter); + doSetCurrentBlock(pBlockIter, pReader->idStr); return TSDB_CODE_SUCCESS; } -static bool blockIteratorNext(SDataBlockIter* pBlockIter) { +static bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr) { bool asc = ASCENDING_TRAVERSE(pBlockIter->order); int32_t step = asc ? 1 : -1; @@ -1189,7 +1194,7 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { } pBlockIter->index += step; - doSetCurrentBlock(pBlockIter); + doSetCurrentBlock(pBlockIter, idStr); return true; } @@ -1260,7 +1265,7 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t ASSERT(pBlockInfo->uid == fblock.uid && pBlockInfo->tbBlockIdx == fblock.tbBlockIdx); } - doSetCurrentBlock(pBlockIter); + doSetCurrentBlock(pBlockIter, ""); return TSDB_CODE_SUCCESS; } @@ -2190,6 +2195,8 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI } static int32_t buildComposedDataBlock(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pResBlock = pReader->pResBlock; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); @@ -2200,6 +2207,13 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pBlockScanInfo == NULL) { + code = TSDB_CODE_INVALID_PARA; + tsdbError("failed to locate the uid:%"PRIu64" in query table uid list, total tables:%d, %s", + pBlockInfo->uid, taosHashGetSize(pReader->status.pTableMap), pReader->idStr); + goto _end; + } + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); @@ -2276,7 +2290,7 @@ _end: pResBlock->info.rows, el, pReader->idStr); } - return TSDB_CODE_SUCCESS; + return code; } void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; } @@ -2732,7 +2746,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { // current block are exhausted, try the next file block if (pDumpInfo->allDumped) { // try next data block in current file - bool hasNext = blockIteratorNext(&pReader->status.blockIter); + bool hasNext = blockIteratorNext(&pReader->status.blockIter, pReader->idStr); if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); } else { @@ -3658,10 +3672,6 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pLReader); } - if (pReader->innerReader[0] != 0) { - tsdbUntakeReadSnap(pReader->innerReader[0]->pTsdb, pReader->innerReader[0]->pReadSnap, pReader->idStr); - } - tsdbDebug( "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 " SMA-time:%.2f ms, fileBlocks:%" PRId64 @@ -3849,8 +3859,14 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { return pReader->pResBlock->pDataBlock; } - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pStatus->blockIter); - STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); + STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pBlockScanInfo == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, + taosHashGetSize(pReader->status.pTableMap), pReader->idStr); + return NULL; + } int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { @@ -3979,7 +3995,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows); pTableBlockInfo->blockRowsHisto[bucketIndex]++; - hasNext = blockIteratorNext(&pStatus->blockIter); + hasNext = blockIteratorNext(&pStatus->blockIter, pReader->idStr); } else { code = initForFirstBlockInFile(pReader, pBlockIter); if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 259c30d591..c6e1ed99f1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -47,7 +47,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { } // do retention - STsdbFS fs; + STsdbFS fs = {0}; code = tsdbFSCopy(pTsdb, &fs); if (code) goto _err; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 6a18ae2fa1..9adef918ba 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -285,15 +285,43 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { + if (offset >= pMsg->contLen) { + qError("vnode offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } + req.msgIdx = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgIdx); + if (offset >= pMsg->contLen) { + qError("vnode offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } + req.msgType = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgType); + if (offset >= pMsg->contLen) { + qError("vnode offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } + req.msgLen = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgLen); + if (offset >= pMsg->contLen) { + qError("vnode offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } + req.msg = (char *)pMsg->pCont + offset; offset += req.msgLen; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1aa3d55efc..01497cc059 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -954,9 +954,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); @@ -980,9 +978,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, - SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode *pAggNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index ebf23ba6bc..1074678efd 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -162,7 +162,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { if (pTableList->map != NULL) { int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t)); - pInfo->pRes->info.groupId = *groupId; + if (groupId != NULL) { + pInfo->pRes->info.groupId = *groupId; + } } else { ASSERT(taosArrayGetSize(pTableList->pTableList) == 1); STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, 0); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index c02b5d2c50..0714d0f3ac 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -234,8 +234,11 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { while (!taosQueueEmpty(pDeleter->pDataBlocks)) { SDataDeleterBuf* pBuf = NULL; taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); - taosMemoryFreeClear(pBuf->pData); - taosFreeQitem(pBuf); + + if (pBuf != NULL) { + taosMemoryFreeClear(pBuf->pData); + taosFreeQitem(pBuf); + } } taosCloseQueue(pDeleter->pDataBlocks); taosThreadMutexDestroy(&pDeleter->mutex); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index d996634d8e..78afdd16b7 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -329,6 +329,8 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat } if (pInserterNode->stableId != suid) { + destroyDataSinker((SDataSinkHandle*)inserter); + taosMemoryFree(inserter); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; return terrno; } @@ -336,6 +338,8 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->pDataBlocks = taosArrayInit(1, POINTER_BYTES); taosThreadMutexInit(&inserter->mutex, NULL); if (NULL == inserter->pDataBlocks) { + destroyDataSinker((SDataSinkHandle*)inserter); + taosMemoryFree(inserter); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index c60c8ff301..cad3e3c44c 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -358,7 +358,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SDataSinkMgtCfg cfg = {.maxDataBlockNum = 10000, .maxDataBlockNumPerQuery = 5000}; code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { - qError("failed to dsDataSinkMgtInit, code: %s", tstrerror(code)); + qError("failed to dsDataSinkMgtInit, code:%s, %s", tstrerror(code), (*pTask)->id.str); goto _error; } @@ -366,7 +366,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, void* pSinkParam = NULL; code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle); if (code != TSDB_CODE_SUCCESS) { - qError("failed to createDataSinkParam, code: %s", tstrerror(code)); + qError("failed to createDataSinkParam, vgId:%d, code:%s, %s", vgId, tstrerror(code), (*pTask)->id.str); goto _error; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index feced8dc60..cb7a644815 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3048,32 +3048,40 @@ void cleanupExprSupp(SExprSupp* pSupp) { taosMemoryFree(pSupp->rowEntryInfoOffset); } -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode,SExecTaskInfo* pTaskInfo) { SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + SSDataBlock* pResBlock = createResDataBlock(pAggNode->node.pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); + int32_t numOfScalarExpr = 0; + SExprInfo* pScalarExprInfo = NULL; + if (pAggNode->pExprs != NULL) { + pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); + } + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 4096); - int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); + int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResultBlock); code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); if (code != TSDB_CODE_SUCCESS) { goto _error; } - pInfo->binfo.mergeResultBlock = mergeResult; + pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; pInfo->groupId = UINT64_MAX; - pInfo->pCondition = pCondition; + pInfo->pCondition = pAggNode->node.pConditions; pOperator->name = "TableAggregate"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_AGG; pOperator->blocking = true; @@ -3332,8 +3340,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT return pTaskInfo; } -static SArray* extractColumnInfo(SNodeList* pNodeList); - SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) { @@ -3710,22 +3716,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createProjectOperatorInfo(ops[0], (SProjectPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_AGG == type) { SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; - SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - - int32_t numOfScalarExpr = 0; - SExprInfo* pScalarExprInfo = NULL; - if (pAggNode->pExprs != NULL) { - pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); - } - if (pAggNode->pGroupKeys != NULL) { - SArray* pColList = extractColumnInfo(pAggNode->pGroupKeys); - pOptr = createGroupOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions, - pScalarExprInfo, numOfScalarExpr, pTaskInfo); + pOptr = createGroupOperatorInfo(ops[0], pAggNode, pTaskInfo); } else { - pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, - pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo); + pOptr = createAggregateOperatorInfo(ops[0], pAggNode, pTaskInfo); } } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; @@ -3815,39 +3809,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOptr; } -SArray* extractColumnInfo(SNodeList* pNodeList) { - size_t numOfCols = LIST_LENGTH(pNodeList); - SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); - if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - for (int32_t i = 0; i < numOfCols; ++i) { - STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); - - if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) { - SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - - SColumn c = extractColumnFromColumnNode(pColNode); - taosArrayPush(pList, &c); - } else if (nodeType(pNode->pExpr) == QUERY_NODE_VALUE) { - SValueNode* pValNode = (SValueNode*)pNode->pExpr; - SColumn c = {0}; - c.slotId = pNode->slotId; - c.colId = pNode->slotId; - c.type = pValNode->node.type; - c.bytes = pValNode->node.resType.bytes; - c.scale = pValNode->node.resType.scale; - c.precision = pValNode->node.resType.precision; - - taosArrayPush(pList, &c); - } - } - - return pList; -} - static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { @@ -4071,6 +4032,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->sql = sql; sql = NULL; + (*pTaskInfo)->pSubplan = pPlan; (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 4ca4ef7b3f..60b11b7326 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -30,6 +30,7 @@ static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDa static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); +static SArray* extractColumnInfo(SNodeList* pNodeList); static void freeGroupKey(void* param) { SGroupKeys* pKey = (SGroupKeys*)param; @@ -61,7 +62,7 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = taosArrayGet(pGroupColList, i); + SColumn* pCol = (SColumn*) taosArrayGet(pGroupColList, i); (*keyLen) += pCol->bytes; // actual data + null_flag SGroupKeys key = {0}; @@ -396,41 +397,48 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { return buildGroupResultDataBlock(pOperator); } -SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, - SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode *pAggNode, SExecTaskInfo* pTaskInfo) { SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pGroupCols = pGroupColList; - pInfo->pCondition = pCondition; + SSDataBlock* pResBlock = createResDataBlock(pAggNode->node.pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); + + int32_t numOfScalarExpr = 0; + SExprInfo* pScalarExprInfo = NULL; + if (pAggNode->pExprs != NULL) { + pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); + } + + pInfo->pGroupCols = extractColumnInfo(pAggNode->pGroupKeys); + pInfo->pCondition = pAggNode->node.pConditions; int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); if (code != TSDB_CODE_SUCCESS) { goto _error; } - code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList); + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); + code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols); if (code != TSDB_CODE_SUCCESS) { goto _error; } initResultSizeInfo(&pOperator->resultInfo, 4096); - code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, pInfo->groupKeyLen, pTaskInfo->id.str); + code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResultBlock); initResultRowInfo(&pInfo->binfo.resultRowInfo); pOperator->name = "GroupbyAggOperator"; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; - // pOperator->operatorType = OP_Groupby; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; @@ -451,8 +459,6 @@ _error: } static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { - // SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SPartitionOperatorInfo* pInfo = pOperator->info; for (int32_t j = 0; j < pBlock->info.rows; ++j) { @@ -760,7 +766,6 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); - pInfo->pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); if (pPartNode->pExprs != NULL) { @@ -781,14 +786,13 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition uint32_t defaultPgsz = 0; uint32_t defaultBufsz = 0; - SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc); - getBufferPgSize(pResBlock->info.rowSize, &defaultPgsz, &defaultBufsz); + pInfo->binfo.pRes = createResDataBlock(pPartNode->node.pOutputDataBlockDesc); + getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz); if (!osTempSpaceAvailable()) { terrno = TSDB_CODE_NO_AVAIL_DISK; pTaskInfo->code = terrno; qError("Create partition operator info failed since %s", terrstr(terrno)); - blockDataDestroy(pResBlock); goto _error; } @@ -797,8 +801,8 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pInfo->rowCapacity = blockDataGetCapacityInRow(pResBlock, getBufPageSize(pInfo->pBuf)); - pInfo->columnOffset = setupColumnOffset(pResBlock, pInfo->rowCapacity); + pInfo->rowCapacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf)); + pInfo->columnOffset = setupColumnOffset(pInfo->binfo.pRes, pInfo->rowCapacity); code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -808,7 +812,6 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; - pInfo->binfo.pRes = pResBlock; pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->info = pInfo; @@ -1102,3 +1105,37 @@ _error: taosMemoryFreeClear(pOperator); return NULL; } + + +SArray* extractColumnInfo(SNodeList* pNodeList) { + size_t numOfCols = LIST_LENGTH(pNodeList); + SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < numOfCols; ++i) { + STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); + + if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) { + SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; + + SColumn c = extractColumnFromColumnNode(pColNode); + taosArrayPush(pList, &c); + } else if (nodeType(pNode->pExpr) == QUERY_NODE_VALUE) { + SValueNode* pValNode = (SValueNode*)pNode->pExpr; + SColumn c = {0}; + c.slotId = pNode->slotId; + c.colId = pNode->slotId; + c.type = pValNode->node.type; + c.bytes = pValNode->node.resType.bytes; + c.scale = pValNode->node.resType.scale; + c.precision = pValNode->node.resType.precision; + + taosArrayPush(pList, &c); + } + } + + return pList; +} diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 53cfa6c27a..eab0307e01 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -59,15 +59,16 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + + int32_t code = TSDB_CODE_SUCCESS; if (pOperator == NULL || pInfo == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } + int32_t numOfCols = 0; SSDataBlock* pResBlock = createResDataBlock(pJoinNode->node.pOutputDataBlockDesc); - - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &numOfCols); - + SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &numOfCols); initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = pResBlock; @@ -84,8 +85,18 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t if (pJoinNode->pOnConditions != NULL && pJoinNode->node.pConditions != NULL) { pInfo->pCondAfterMerge = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (pInfo->pCondAfterMerge == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pInfo->pCondAfterMerge); pLogicCond->pParameterList = nodesMakeList(); + if (pLogicCond->pParameterList == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->pOnConditions)); nodesListMakeAppend(&pLogicCond->pParameterList, nodesCloneNode(pJoinNode->node.pConditions)); pLogicCond->condType = LOGIC_COND_TYPE_AND; @@ -106,7 +117,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL, NULL, NULL); - int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); + code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -114,9 +125,12 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t return pOperator; _error: - taosMemoryFree(pInfo); + if (pInfo != NULL) { + destroyMergeJoinOperator(pInfo); + } + taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 058b9b1b44..cdc59deee1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -494,7 +494,13 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* p SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData}; SScalarParam param = {.columnData = pColInfoData}; - fpSet.process(&srcParam, 1, ¶m); + + if (fpSet.process != NULL) { + fpSet.process(&srcParam, 1, ¶m); + } else { + qError("failed to get the corresponding callback function, functionId:%d", functionId); + } + colDataDestroy(&infoData); } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 7ca3de5214..c00b5c4802 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -719,12 +719,16 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; - pInfo->binfo.pRes = createResDataBlock(pDescNode); - int32_t rowSize = pInfo->binfo.pRes->info.rowSize; - if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { + int32_t code = TSDB_CODE_SUCCESS; + if (pInfo == NULL || pOperator == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } + pInfo->binfo.pRes = createResDataBlock(pDescNode); + int32_t rowSize = pInfo->binfo.pRes->info.rowSize; + ASSERT(rowSize < 100 * 1024 * 1024); + SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys); int32_t numOfOutputCols = 0; SArray* pColMatchColInfo = @@ -737,6 +741,9 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->pSortInfo = pSortInfo; pInfo->pColMatchInfo = pColMatchColInfo; pInfo->pInputBlock = pInputBlock; + pInfo->bufPageSize = getProperSortPageSize(rowSize); + pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. + pOperator->name = "MultiwayMerge"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; pOperator->blocking = false; @@ -744,15 +751,10 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pInfo->bufPageSize = getProperSortPageSize(rowSize); - - // one additional is reserved for merged result. - pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); - pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL, destroyMultiwayMergeOperatorInfo, NULL, NULL, getMultiwayMergeExplainExecInfo); - int32_t code = appendDownstream(pOperator, downStreams, numStreams); + code = appendDownstream(pOperator, downStreams, numStreams); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 505654d967..15cad81d23 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3594,9 +3594,7 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT pKey->win.skey = startTs; pKey->win.ekey = endTs; pKey->groupId = groupId; - SStreamStateCur* pCur = streamStateSessionGetCur(pAggSup->pState, pKey); - int32_t code = streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); - streamStateFreeCur(pCur); + int32_t code = streamStateSessionGetKey(pAggSup->pState, pKey, pKey); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index b133041fdc..4204692514 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -251,6 +251,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ if (!osTempSpaceAvailable()) { terrno = TSDB_CODE_NO_AVAIL_DISK; printf("tHash Init failed since %s", terrstr(terrno)); + taosMemoryFree(pHashObj); return NULL; } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 03248f5069..7ad5c1365c 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -140,6 +140,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSource int32_t* sourceId, SArray* pPageIdList) { SSortSource* pSource = taosMemoryCalloc(1, sizeof(SSortSource)); if (pSource == NULL) { + taosArrayDestroy(pPageIdList); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -155,6 +156,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSource int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock))) / rowSize; ASSERT(numOfRows > 0); + return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); } @@ -224,6 +226,22 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int int32_t code = 0; + // multi-pass internal merge sort is required + if (pHandle->pBuf == NULL) { + if (!osTempSpaceAvailable()) { + code = TSDB_CODE_NO_AVAIL_DISK; + qError("Sort compare init failed since %s", terrstr(code)); + return code; + } + + code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, + "sortComparInit", tsTempDir); + dBufSetPrintInfo(pHandle->pBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + if (pHandle->type == SORT_SINGLESOURCE_SORT) { for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { SSortSource* pSource = cmpParam->pSources[i]; @@ -245,22 +263,6 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int releaseBufPage(pHandle->pBuf, pPage); } } else { - // multi-pass internal merge sort is required - if (pHandle->pBuf == NULL) { - if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - code = terrno; - qError("Sort compare init failed since %s", terrstr(terrno)); - return code; - } - code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, - "sortComparInit", tsTempDir); - dBufSetPrintInfo(pHandle->pBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { SSortSource* pSource = cmpParam->pSources[i]; pSource->src.pBlock = pHandle->fetchfp(pSource->param); @@ -507,12 +509,14 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { int32_t code = sortComparInit(&pHandle->cmpParam, pHandle->pOrderedSource, i * numOfInputSources, end, pHandle); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pResList); return code; } code = tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pResList); return code; } @@ -520,12 +524,16 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { while (1) { SSDataBlock* pDataBlock = getSortedBlockDataInner(pHandle, &pHandle->cmpParam, numOfRows); if (pDataBlock == NULL) { + taosArrayDestroy(pResList); + taosArrayDestroy(pPageIdList); break; } int32_t pageId = -1; void* pPage = getNewBufPage(pHandle->pBuf, &pageId); if (pPage == NULL) { + taosArrayDestroy(pResList); + taosArrayDestroy(pPageIdList); return terrno; } diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index c1b0941d4b..c69d19b8a6 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -35,7 +35,7 @@ enum { }; typedef struct SUdfSetupRequest { - char udfName[TSDB_FUNC_NAME_LEN]; + char udfName[TSDB_FUNC_NAME_LEN + 1]; } SUdfSetupRequest; typedef struct SUdfSetupResponse { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 49b2168dff..d107ef2ac8 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1163,13 +1163,13 @@ static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, c // the data is loaded, not only the block SMA value for (int32_t i = start; i < num + start; ++i) { char* p = colDataGetData(pCol, i); - if (memcpy((void*)tval, p, pCol->info.bytes) == 0) { + if (memcmp((void*)tval, p, pCol->info.bytes) == 0) { return i; } } - ASSERT(0); - return 0; + // if reach here means real data of block SMA is not set in pCtx->input. + return -1; } int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { @@ -1211,7 +1211,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = *(int64_t*)tval; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + if (index >= 0) { + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + } } } else { if (IS_SIGNED_NUMERIC_TYPE(type)) { @@ -1223,7 +1225,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { *(int64_t*)&pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + if (index >= 0) { + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + } } } } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { @@ -1235,7 +1239,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { *(uint64_t*)&pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + if (index >= 0) { + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + } } } } else if (type == TSDB_DATA_TYPE_DOUBLE) { @@ -1247,7 +1253,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { *(double*)&pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + if (index >= 0) { + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + } } } } else if (type == TSDB_DATA_TYPE_FLOAT) { @@ -1261,7 +1269,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + if (index >= 0) { + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); + } } } } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5b8389078f..e2e508a9b3 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -315,7 +315,7 @@ enum { UV_TASK_CONNECT = 0, UV_TASK_REQ_RSP = 1, UV_TASK_DISCONNECT = 2 }; int64_t gUdfTaskSeqNum = 0; typedef struct SUdfcFuncStub { - char udfName[TSDB_FUNC_NAME_LEN]; + char udfName[TSDB_FUNC_NAME_LEN + 1]; UdfcFuncHandle handle; int32_t refCount; int64_t lastRefTime; @@ -353,7 +353,7 @@ typedef struct SUdfcUvSession { int32_t outputLen; int32_t bufSize; - char udfName[TSDB_FUNC_NAME_LEN]; + char udfName[TSDB_FUNC_NAME_LEN + 1]; } SUdfcUvSession; typedef struct SClientUvTaskNode { @@ -898,7 +898,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { int32_t code = 0; uv_mutex_lock(&gUdfdProxy.udfStubsMutex); SUdfcFuncStub key = {0}; - strcpy(key.udfName, udfName); + strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); int32_t stubIndex = taosArraySearchIdx(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); if (stubIndex != -1) { SUdfcFuncStub *foundStub = taosArrayGet(gUdfdProxy.udfStubs, stubIndex); @@ -936,7 +936,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { void releaseUdfFuncHandle(char *udfName) { uv_mutex_lock(&gUdfdProxy.udfStubsMutex); SUdfcFuncStub key = {0}; - strcpy(key.udfName, udfName); + strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); if (!foundStub) { uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); @@ -1446,6 +1446,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { QUEUE_INSERT_TAIL(connTaskQueue, &uvTask->connTaskQueue); int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipeWrite); if (err != 0) { + taosMemoryFree(write); fnError("udfc event loop start req_rsp task uv_write failed. uvtask: %p, code: %s", uvTask, uv_strerror(err)); } code = err; @@ -1637,7 +1638,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { task->session->outputType = rsp->outputType; task->session->outputLen = rsp->outputLen; task->session->bufSize = rsp->bufSize; - strcpy(task->session->udfName, udfName); + strncpy(task->session->udfName, udfName, TSDB_FUNC_NAME_LEN); if (task->errCode != 0) { fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode) } else { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index a319b5a25b..f8c4f171ba 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -71,7 +71,7 @@ typedef struct SUdf { uv_cond_t condReady; bool resident; - char name[TSDB_FUNC_NAME_LEN]; + char name[TSDB_FUNC_NAME_LEN + 1]; int8_t funcType; int8_t scriptType; int8_t outputType; @@ -188,11 +188,12 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); udfNew->refCount = 1; udfNew->state = UDF_STATE_INIT; - uv_mutex_init(&udfNew->lock); uv_cond_init(&udfNew->condReady); + udf = udfNew; - taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), &udfNew, sizeof(&udfNew)); + SUdf** pUdf = &udf; + taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), pUdf, POINTER_BYTES); uv_mutex_unlock(&global.udfsMutex); } @@ -246,7 +247,7 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { SUdfCallRequest *call = &request->call; - fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request->seqNum, call->callType, call->udfHandle); + fnDebug("call request. call type %d, handle: %" PRIx64 ", seq num %" PRId64 , call->callType, call->udfHandle, request->seqNum); SUdfcFuncHandle * handle = (SUdfcFuncHandle *)(call->udfHandle); SUdf * udf = handle->udf; SUdfResponse response = {0}; @@ -372,7 +373,7 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { } taosMemoryFree(handle); - SUdfResponse response; + SUdfResponse response = {0}; SUdfResponse *rsp = &response; rsp->seqNum = request->seqNum; rsp->type = request->type; @@ -428,7 +429,9 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { } else if (msgInfo->rpcType == UDFD_RPC_RETRIVE_FUNC) { SRetrieveFuncRsp retrieveRsp = {0}; tDeserializeSRetrieveFuncRsp(pMsg->pCont, pMsg->contLen, &retrieveRsp); - + if (retrieveRsp.pFuncInfos == NULL) { + goto _return; + } SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); SUdf * udf = msgInfo->param; udf->funcType = pFuncInfo->funcType; @@ -540,7 +543,7 @@ int32_t udfdConnectToMnode() { } int32_t udfdLoadUdf(char *udfName, SUdf *udf) { - strcpy(udf->name, udfName); + strncpy(udf->name, udfName, TSDB_FUNC_NAME_LEN); int32_t err = 0; err = udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf); @@ -880,6 +883,8 @@ static int32_t udfdUvInit() { uv_loop_t *loop = taosMemoryMalloc(sizeof(uv_loop_t)); if (loop) { uv_loop_init(loop); + } else { + return -1; } global.loop = loop; @@ -901,12 +906,12 @@ static int32_t udfdUvInit() { if ((r = uv_pipe_bind(&global.listeningPipe, global.listenPipeName))) { fnError("Bind error %s", uv_err_name(r)); removeListeningPipe(); - return -1; + return -2; } if ((r = uv_listen((uv_stream_t *)&global.listeningPipe, 128, udfdOnNewConnection))) { fnError("Listen error %s", uv_err_name(r)); removeListeningPipe(); - return -2; + return -3; } return 0; } @@ -962,6 +967,7 @@ int32_t udfdInitResidentFuncs() { while ((token = strtok_r(pSave, ",", &pSave)) != NULL) { char func[TSDB_FUNC_NAME_LEN+1] = {0}; strncpy(func, token, TSDB_FUNC_NAME_LEN); + fnInfo("udfd add resident function %s", func); taosArrayPush(global.residentFuncs, func); } diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index f1e3f4c60c..b5d1b1576b 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -110,8 +110,11 @@ int aggregateFuncTest() { taosArrayDestroy(pBlock->pDataBlock); doCallUdfAggFinalize(handle, &newBuf, &resultBuf); - fprintf(stderr, "agg result: %f\n", *(double *)resultBuf.buf); - + if (resultBuf.buf != NULL) { + fprintf(stderr, "agg result: %f\n", *(double *)resultBuf.buf); + } else { + fprintf(stderr, "result buffer is null"); + } freeUdfInterBuf(&buf); freeUdfInterBuf(&newBuf); freeUdfInterBuf(&resultBuf); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 25036996fc..e3f140047a 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -367,7 +367,7 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(dst, src, strlen(src)); break; default: - TASSERT(0); + ASSERT(0); break; } *dst = (char*)*dst - tlen; @@ -459,7 +459,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { *dst = (char*)*dst - tlen; break; default: - TASSERT(0); + ASSERT(0); break; } return tlen; diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 890a59e4be..70792ee611 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -450,17 +450,10 @@ static void monGenDiskJson(SMonInfo *pMonitor) { } static const char *monLogLevelStr(ELogLevel level) { - switch (level) { - case DEBUG_ERROR: - return "error"; - case DEBUG_INFO: - return "info"; - case DEBUG_DEBUG: - return "debug"; - case DEBUG_TRACE: - return "trace"; - default: - return "undefine"; + if (level == DEBUG_ERROR) { + return "error"; + } else { + return "info"; } } diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index dd371f874c..c20055067e 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -18,272 +18,6 @@ #include "tcoding.h" #include "tencode.h" -static int32_t tEncodeSMonSysInfo(SEncoder *encoder, const SMonSysInfo *pInfo) { - if (tEncodeDouble(encoder, pInfo->cpu_engine) < 0) return -1; - if (tEncodeDouble(encoder, pInfo->cpu_system) < 0) return -1; - if (tEncodeFloat(encoder, pInfo->cpu_cores) < 0) return -1; - if (tEncodeI64(encoder, pInfo->mem_engine) < 0) return -1; - if (tEncodeI64(encoder, pInfo->mem_system) < 0) return -1; - if (tEncodeI64(encoder, pInfo->mem_total) < 0) return -1; - if (tEncodeI64(encoder, pInfo->disk_engine) < 0) return -1; - if (tEncodeI64(encoder, pInfo->disk_used) < 0) return -1; - if (tEncodeI64(encoder, pInfo->disk_total) < 0) return -1; - if (tEncodeI64(encoder, pInfo->net_in) < 0) return -1; - if (tEncodeI64(encoder, pInfo->net_out) < 0) return -1; - if (tEncodeI64(encoder, pInfo->io_read) < 0) return -1; - if (tEncodeI64(encoder, pInfo->io_write) < 0) return -1; - if (tEncodeI64(encoder, pInfo->io_read_disk) < 0) return -1; - if (tEncodeI64(encoder, pInfo->io_write_disk) < 0) return -1; - return 0; -} - -static int32_t tDecodeSMonSysInfo(SDecoder *decoder, SMonSysInfo *pInfo) { - if (tDecodeDouble(decoder, &pInfo->cpu_engine) < 0) return -1; - if (tDecodeDouble(decoder, &pInfo->cpu_system) < 0) return -1; - if (tDecodeFloat(decoder, &pInfo->cpu_cores) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->mem_engine) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->mem_system) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->mem_total) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->disk_engine) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->disk_used) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->disk_total) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->net_in) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->net_out) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->io_read) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->io_write) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->io_read_disk) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->io_write_disk) < 0) return -1; - return 0; -} - -int32_t tEncodeSMonLogs(SEncoder *encoder, const SMonLogs *pInfo) { - if (tEncodeI32(encoder, pInfo->numOfErrorLogs) < 0) return -1; - if (tEncodeI32(encoder, pInfo->numOfInfoLogs) < 0) return -1; - if (tEncodeI32(encoder, pInfo->numOfDebugLogs) < 0) return -1; - if (tEncodeI32(encoder, pInfo->numOfTraceLogs) < 0) return -1; - if (tEncodeI32(encoder, taosArrayGetSize(pInfo->logs)) < 0) return -1; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->logs); ++i) { - SMonLogItem *pLog = taosArrayGet(pInfo->logs, i); - if (tEncodeI64(encoder, pLog->ts) < 0) return -1; - if (tEncodeI8(encoder, pLog->level) < 0) return -1; - if (tEncodeCStr(encoder, pLog->content) < 0) return -1; - } - return 0; -} - -static int32_t tDecodeSMonLogs(SDecoder *decoder, SMonLogs *pInfo) { - if (tDecodeI32(decoder, &pInfo->numOfErrorLogs) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->numOfInfoLogs) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->numOfDebugLogs) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->numOfTraceLogs) < 0) return -1; - - int32_t arraySize = 0; - if (tDecodeI32(decoder, &arraySize) < 0) return -1; - - pInfo->logs = taosArrayInit(arraySize, sizeof(SMonLogItem)); - if (pInfo->logs == NULL) return -1; - - for (int32_t i = 0; i < arraySize; ++i) { - SMonLogItem desc = {0}; - if (tDecodeI64(decoder, &desc.ts) < 0) return -1; - int8_t level = 0; - if (tDecodeI8(decoder, &level) < 0) return -1; - desc.level = level; - if (tDecodeCStrTo(decoder, desc.content) < 0) return -1; - taosArrayPush(pInfo->logs, &desc); - } - - return 0; -} - -int32_t tEncodeSMonClusterInfo(SEncoder *encoder, const SMonClusterInfo *pInfo) { - if (tEncodeCStr(encoder, pInfo->first_ep) < 0) return -1; - if (tEncodeI32(encoder, pInfo->first_ep_dnode_id) < 0) return -1; - if (tEncodeCStr(encoder, pInfo->version) < 0) return -1; - if (tEncodeFloat(encoder, pInfo->master_uptime) < 0) return -1; - if (tEncodeI32(encoder, pInfo->monitor_interval) < 0) return -1; - if (tEncodeI32(encoder, pInfo->dbs_total) < 0) return -1; - if (tEncodeI32(encoder, pInfo->stbs_total) < 0) return -1; - if (tEncodeI64(encoder, pInfo->tbs_total) < 0) return -1; - if (tEncodeI32(encoder, pInfo->vgroups_total) < 0) return -1; - if (tEncodeI32(encoder, pInfo->vgroups_alive) < 0) return -1; - if (tEncodeI32(encoder, pInfo->vnodes_total) < 0) return -1; - if (tEncodeI32(encoder, pInfo->vnodes_alive) < 0) return -1; - if (tEncodeI32(encoder, pInfo->connections_total) < 0) return -1; - if (tEncodeI32(encoder, taosArrayGetSize(pInfo->dnodes)) < 0) return -1; - if (tEncodeI32(encoder, taosArrayGetSize(pInfo->mnodes)) < 0) return -1; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { - SMonDnodeDesc *pDesc = taosArrayGet(pInfo->dnodes, i); - if (tEncodeI32(encoder, pDesc->dnode_id) < 0) return -1; - if (tEncodeCStr(encoder, pDesc->dnode_ep) < 0) return -1; - if (tEncodeCStr(encoder, pDesc->status) < 0) return -1; - } - for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { - SMonMnodeDesc *pDesc = taosArrayGet(pInfo->mnodes, i); - if (tEncodeI32(encoder, pDesc->mnode_id) < 0) return -1; - if (tEncodeCStr(encoder, pDesc->mnode_ep) < 0) return -1; - if (tEncodeCStr(encoder, pDesc->role) < 0) return -1; - } - return 0; -} - -int32_t tDecodeSMonClusterInfo(SDecoder *decoder, SMonClusterInfo *pInfo) { - if (tDecodeCStrTo(decoder, pInfo->first_ep) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->first_ep_dnode_id) < 0) return -1; - if (tDecodeCStrTo(decoder, pInfo->version) < 0) return -1; - if (tDecodeFloat(decoder, &pInfo->master_uptime) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->monitor_interval) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->dbs_total) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->stbs_total) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->tbs_total) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->vgroups_total) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->vgroups_alive) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->vnodes_total) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->vnodes_alive) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->connections_total) < 0) return -1; - - int32_t dnodesSize = 0; - int32_t mnodesSize = 0; - if (tDecodeI32(decoder, &dnodesSize) < 0) return -1; - if (tDecodeI32(decoder, &mnodesSize) < 0) return -1; - - pInfo->dnodes = taosArrayInit(dnodesSize, sizeof(SMonDnodeDesc)); - pInfo->mnodes = taosArrayInit(mnodesSize, sizeof(SMonMnodeDesc)); - if (pInfo->dnodes == NULL || pInfo->mnodes == NULL) return -1; - - for (int32_t i = 0; i < dnodesSize; ++i) { - SMonDnodeDesc desc = {0}; - if (tDecodeI32(decoder, &desc.dnode_id) < 0) return -1; - if (tDecodeCStrTo(decoder, desc.dnode_ep) < 0) return -1; - if (tDecodeCStrTo(decoder, desc.status) < 0) return -1; - taosArrayPush(pInfo->dnodes, &desc); - } - - for (int32_t i = 0; i < mnodesSize; ++i) { - SMonMnodeDesc desc = {0}; - if (tDecodeI32(decoder, &desc.mnode_id) < 0) return -1; - if (tDecodeCStrTo(decoder, desc.mnode_ep) < 0) return -1; - if (tDecodeCStrTo(decoder, desc.role) < 0) return -1; - taosArrayPush(pInfo->mnodes, &desc); - } - return 0; -} - -int32_t tEncodeSMonVgroupInfo(SEncoder *encoder, const SMonVgroupInfo *pInfo) { - if (tEncodeI32(encoder, taosArrayGetSize(pInfo->vgroups)) < 0) return -1; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { - SMonVgroupDesc *pDesc = taosArrayGet(pInfo->vgroups, i); - if (tEncodeI32(encoder, pDesc->vgroup_id) < 0) return -1; - if (tEncodeI32(encoder, pDesc->tables_num) < 0) return -1; - if (tEncodeCStr(encoder, pDesc->database_name) < 0) return -1; - if (tEncodeCStr(encoder, pDesc->status) < 0) return -1; - for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { - SMonVnodeDesc *pVDesc = &pDesc->vnodes[j]; - if (tEncodeI32(encoder, pVDesc->dnode_id) < 0) return -1; - if (tEncodeCStr(encoder, pVDesc->vnode_role) < 0) return -1; - } - } - return 0; -} - -int32_t tDecodeSMonVgroupInfo(SDecoder *decoder, SMonVgroupInfo *pInfo) { - int32_t arraySize = 0; - if (tDecodeI32(decoder, &arraySize) < 0) return -1; - - pInfo->vgroups = taosArrayInit(arraySize, sizeof(SMonVgroupDesc)); - if (pInfo->vgroups == NULL) return -1; - - for (int32_t i = 0; i < arraySize; ++i) { - SMonVgroupDesc desc = {0}; - if (tDecodeI32(decoder, &desc.vgroup_id) < 0) return -1; - if (tDecodeI32(decoder, &desc.tables_num) < 0) return -1; - if (tDecodeCStrTo(decoder, desc.database_name) < 0) return -1; - if (tDecodeCStrTo(decoder, desc.status) < 0) return -1; - for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { - SMonVnodeDesc *pVDesc = &desc.vnodes[j]; - if (tDecodeI32(decoder, &pVDesc->dnode_id) < 0) return -1; - if (tDecodeCStrTo(decoder, pVDesc->vnode_role) < 0) return -1; - } - taosArrayPush(pInfo->vgroups, &desc); - } - return 0; -} - -int32_t tEncodeSMonStbInfo(SEncoder *encoder, const SMonStbInfo *pInfo) { - if (tEncodeI32(encoder, taosArrayGetSize(pInfo->stbs)) < 0) return -1; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->stbs); ++i) { - SMonStbDesc *pDesc = taosArrayGet(pInfo->stbs, i); - if (tEncodeCStr(encoder, pDesc->stb_name) < 0) return -1; - if (tEncodeCStr(encoder, pDesc->database_name) < 0) return -1; - } - return 0; -} - -int32_t tDecodeSMonStbInfo(SDecoder *decoder, SMonStbInfo *pInfo) { - int32_t arraySize = 0; - if (tDecodeI32(decoder, &arraySize) < 0) return -1; - - pInfo->stbs = taosArrayInit(arraySize, sizeof(SMonStbDesc)); - if (pInfo->stbs == NULL) return -1; - - for (int32_t i = 0; i < arraySize; ++i) { - SMonStbDesc desc = {0}; - if (tDecodeCStrTo(decoder, desc.stb_name) < 0) return -1; - if (tDecodeCStrTo(decoder, desc.database_name) < 0) return -1; - taosArrayPush(pInfo->stbs, &desc); - } - return 0; -} - -int32_t tEncodeSMonGrantInfo(SEncoder *encoder, const SMonGrantInfo *pInfo) { - if (tEncodeI32(encoder, pInfo->expire_time) < 0) return -1; - if (tEncodeI64(encoder, pInfo->timeseries_used) < 0) return -1; - if (tEncodeI64(encoder, pInfo->timeseries_total) < 0) return -1; - return 0; -} - -int32_t tDecodeSMonGrantInfo(SDecoder *decoder, SMonGrantInfo *pInfo) { - if (tDecodeI32(decoder, &pInfo->expire_time) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->timeseries_used) < 0) return -1; - if (tDecodeI64(decoder, &pInfo->timeseries_total) < 0) return -1; - return 0; -} - -int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeSMonClusterInfo(&encoder, &pInfo->cluster) < 0) return -1; - if (tEncodeSMonVgroupInfo(&encoder, &pInfo->vgroup) < 0) return -1; - if (tEncodeSMonStbInfo(&encoder, &pInfo->stb) < 0) return -1; - if (tEncodeSMonGrantInfo(&encoder, &pInfo->grant) < 0) return -1; - if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; - if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeSMonClusterInfo(&decoder, &pInfo->cluster) < 0) return -1; - if (tDecodeSMonVgroupInfo(&decoder, &pInfo->vgroup) < 0) return -1; - if (tDecodeSMonStbInfo(&decoder, &pInfo->stb) < 0) return -1; - if (tDecodeSMonGrantInfo(&decoder, &pInfo->grant) < 0) return -1; - if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; - if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - void tFreeSMonMmInfo(SMonMmInfo *pInfo) { taosArrayDestroy(pInfo->log.logs); taosArrayDestroy(pInfo->cluster.mnodes); @@ -297,106 +31,6 @@ void tFreeSMonMmInfo(SMonMmInfo *pInfo) { pInfo->log.logs = NULL; } -int32_t tEncodeSMonDiskDesc(SEncoder *encoder, const SMonDiskDesc *pDesc) { - if (tEncodeCStr(encoder, pDesc->name) < 0) return -1; - if (tEncodeI8(encoder, pDesc->level) < 0) return -1; - if (tEncodeI64(encoder, pDesc->size.total) < 0) return -1; - if (tEncodeI64(encoder, pDesc->size.used) < 0) return -1; - if (tEncodeI64(encoder, pDesc->size.avail) < 0) return -1; - return 0; -} - -static int32_t tDecodeSMonDiskDesc(SDecoder *decoder, SMonDiskDesc *pDesc) { - if (tDecodeCStrTo(decoder, pDesc->name) < 0) return -1; - if (tDecodeI8(decoder, &pDesc->level) < 0) return -1; - if (tDecodeI64(decoder, &pDesc->size.total) < 0) return -1; - if (tDecodeI64(decoder, &pDesc->size.used) < 0) return -1; - if (tDecodeI64(decoder, &pDesc->size.avail) < 0) return -1; - return 0; -} - -int32_t tEncodeSMonDiskInfo(SEncoder *encoder, const SMonDiskInfo *pInfo) { - if (tEncodeI32(encoder, taosArrayGetSize(pInfo->datadirs)) < 0) return -1; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->datadirs); ++i) { - SMonDiskDesc *pDesc = taosArrayGet(pInfo->datadirs, i); - if (tEncodeSMonDiskDesc(encoder, pDesc) < 0) return -1; - } - return 0; -} - -static int32_t tDecodeSMonDiskInfo(SDecoder *decoder, SMonDiskInfo *pInfo) { - int32_t arraySize = 0; - if (tDecodeI32(decoder, &arraySize) < 0) return -1; - - pInfo->datadirs = taosArrayInit(arraySize, sizeof(SMonDiskDesc)); - if (pInfo->datadirs == NULL) return -1; - - for (int32_t i = 0; i < arraySize; ++i) { - SMonDiskDesc desc = {0}; - if (tDecodeSMonDiskDesc(decoder, &desc) < 0) return -1; - taosArrayPush(pInfo->datadirs, &desc); - } - - return 0; -} - -int32_t tEncodeSVnodesStat(SEncoder *encoder, const SVnodesStat *pStat) { - if (tEncodeI32(encoder, pStat->openVnodes) < 0) return -1; - if (tEncodeI32(encoder, pStat->totalVnodes) < 0) return -1; - if (tEncodeI32(encoder, pStat->masterNum) < 0) return -1; - if (tEncodeI64(encoder, pStat->numOfSelectReqs) < 0) return -1; - if (tEncodeI64(encoder, pStat->numOfInsertReqs) < 0) return -1; - if (tEncodeI64(encoder, pStat->numOfInsertSuccessReqs) < 0) return -1; - if (tEncodeI64(encoder, pStat->numOfBatchInsertReqs) < 0) return -1; - if (tEncodeI64(encoder, pStat->numOfBatchInsertSuccessReqs) < 0) return -1; - if (tEncodeI64(encoder, pStat->errors) < 0) return -1; - return 0; -} - -static int32_t tDecodeSVnodesStat(SDecoder *decoder, SVnodesStat *pStat) { - if (tDecodeI32(decoder, &pStat->openVnodes) < 0) return -1; - if (tDecodeI32(decoder, &pStat->totalVnodes) < 0) return -1; - if (tDecodeI32(decoder, &pStat->masterNum) < 0) return -1; - if (tDecodeI64(decoder, &pStat->numOfSelectReqs) < 0) return -1; - if (tDecodeI64(decoder, &pStat->numOfInsertReqs) < 0) return -1; - if (tDecodeI64(decoder, &pStat->numOfInsertSuccessReqs) < 0) return -1; - if (tDecodeI64(decoder, &pStat->numOfBatchInsertReqs) < 0) return -1; - if (tDecodeI64(decoder, &pStat->numOfBatchInsertSuccessReqs) < 0) return -1; - if (tDecodeI64(decoder, &pStat->errors) < 0) return -1; - return 0; -} - -int32_t tSerializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeSMonDiskInfo(&encoder, &pInfo->tfs) < 0) return -1; - if (tEncodeSVnodesStat(&encoder, &pInfo->vstat) < 0) return -1; - if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; - if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeSMonDiskInfo(&decoder, &pInfo->tfs) < 0) return -1; - if (tDecodeSVnodesStat(&decoder, &pInfo->vstat) < 0) return -1; - if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; - if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - void tFreeSMonVmInfo(SMonVmInfo *pInfo) { taosArrayDestroy(pInfo->log.logs); taosArrayDestroy(pInfo->tfs.datadirs); @@ -404,243 +38,17 @@ void tFreeSMonVmInfo(SMonVmInfo *pInfo) { pInfo->tfs.datadirs = NULL; } -int32_t tSerializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; - if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; - if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - void tFreeSMonQmInfo(SMonQmInfo *pInfo) { taosArrayDestroy(pInfo->log.logs); pInfo->log.logs = NULL; } -int32_t tSerializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; - if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; - if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - void tFreeSMonSmInfo(SMonSmInfo *pInfo) { taosArrayDestroy(pInfo->log.logs); pInfo->log.logs = NULL; } -int32_t tSerializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; - if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; - if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - void tFreeSMonBmInfo(SMonBmInfo *pInfo) { taosArrayDestroy(pInfo->log.logs); pInfo->log.logs = NULL; } - -int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, taosArrayGetSize(pInfo->pVloads)) < 0) return -1; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->pVloads); ++i) { - SVnodeLoad *pLoad = taosArrayGet(pInfo->pVloads, i); - if (tEncodeI32(&encoder, pLoad->vgId) < 0) return -1; - if (tEncodeI8(&encoder, pLoad->syncState) < 0) return -1; - if (tEncodeI8(&encoder, pLoad->syncRestore) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->cacheUsage) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->numOfTables) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->numOfTimeSeries) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->totalStorage) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->compStorage) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->pointsWritten) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->numOfSelectReqs) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->numOfInsertReqs) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->numOfInsertSuccessReqs) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->numOfBatchInsertReqs) < 0) return -1; - if (tEncodeI64(&encoder, pLoad->numOfBatchInsertSuccessReqs) < 0) return -1; - } - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - - int32_t arraySize = 0; - if (tDecodeI32(&decoder, &arraySize) < 0) return -1; - - pInfo->pVloads = taosArrayInit(arraySize, sizeof(SVnodeLoad)); - if (pInfo->pVloads == NULL) return -1; - - for (int32_t i = 0; i < arraySize; ++i) { - SVnodeLoad load = {0}; - if (tDecodeI32(&decoder, &load.vgId) < 0) return -1; - if (tDecodeI8(&decoder, &load.syncState) < 0) return -1; - if (tDecodeI8(&decoder, &load.syncRestore) < 0) return -1; - if (tDecodeI64(&decoder, &load.cacheUsage) < 0) return -1; - if (tDecodeI64(&decoder, &load.numOfTables) < 0) return -1; - if (tDecodeI64(&decoder, &load.numOfTimeSeries) < 0) return -1; - if (tDecodeI64(&decoder, &load.totalStorage) < 0) return -1; - if (tDecodeI64(&decoder, &load.compStorage) < 0) return -1; - if (tDecodeI64(&decoder, &load.pointsWritten) < 0) return -1; - if (tDecodeI64(&decoder, &load.numOfSelectReqs) < 0) return -1; - if (tDecodeI64(&decoder, &load.numOfInsertReqs) < 0) return -1; - if (tDecodeI64(&decoder, &load.numOfInsertSuccessReqs) < 0) return -1; - if (tDecodeI64(&decoder, &load.numOfBatchInsertReqs) < 0) return -1; - if (tDecodeI64(&decoder, &load.numOfBatchInsertSuccessReqs) < 0) return -1; - taosArrayPush(pInfo->pVloads, &load); - } - - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} - -void tFreeSMonVloadInfo(SMonVloadInfo *pInfo) { - taosArrayDestroy(pInfo->pVloads); - pInfo->pVloads = NULL; -} - -int32_t tSerializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI8(&encoder, pInfo->isMnode) < 0) return -1; - if (tEncodeI8(&encoder, pInfo->load.syncState) < 0) return -1; - if (tEncodeI8(&encoder, pInfo->load.syncRestore) < 0) return -1; - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInfo) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI8(&decoder, &pInfo->isMnode) < 0) return -1; - if (tDecodeI8(&decoder, &pInfo->load.syncState) < 0) return -1; - if (tDecodeI8(&decoder, &pInfo->load.syncRestore) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} - -int32_t tSerializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->numOfProcessedQuery) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->numOfProcessedCQuery) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->numOfProcessedFetch) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->numOfProcessedDrop) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->numOfProcessedHb) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->numOfProcessedDelete) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->cacheDataSize) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->numOfQueryInQueue) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->numOfFetchInQueue) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->timeInQueryQueue) < 0) return -1; - if (tEncodeI64(&encoder, pInfo->timeInFetchQueue) < 0) return -1; - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->numOfProcessedQuery) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->numOfProcessedCQuery) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->numOfProcessedFetch) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->numOfProcessedDrop) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->numOfProcessedHb) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->numOfProcessedDelete) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->cacheDataSize) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->numOfQueryInQueue) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->numOfFetchInQueue) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->timeInQueryQueue) < 0) return -1; - if (tDecodeI64(&decoder, &pInfo->timeInFetchQueue) < 0) return -1; - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; -} diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 339065633b..1d8744d726 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1178,9 +1178,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); - int64_t factor = - (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); - + int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int64_t unit = timeUnit * 1000 / factor; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { @@ -1372,9 +1370,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); } - int64_t factor = - (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); - + int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int32_t numOfRows = 0; for (int32_t i = 0; i < inputNum; ++i) { if (pInput[i].numOfRows > numOfRows) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f089bad04e..6480c53dbf 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1426,57 +1426,6 @@ void vectorAssign(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, pOut->numOfQualified = pRight->numOfQualified * pOut->numOfRows; } -void vectorConcat(SScalarParam *pLeft, SScalarParam *pRight, void *out, int32_t _ord) { -#if 0 - int32_t len = pLeft->bytes + pRight->bytes; - - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - char *output = (char *)out; - if (pLeft->numOfRows == pRight->numOfRows) { - for (; i < pRight->numOfRows && i >= 0; i += step, output += len) { - char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); - char* right = POINTER_SHIFT(pRight->data, pRight->bytes * i); - - if (isNull(left, pLeftCol->info.type) || isNull(right, pRight->info.type)) { - setVardataNull(output, TSDB_DATA_TYPE_BINARY); - continue; - } - - // todo define a macro - memcpy(varDataVal(output), varDataVal(left), varDataLen(left)); - memcpy(varDataVal(output) + varDataLen(left), varDataVal(right), varDataLen(right)); - varDataSetLen(output, varDataLen(left) + varDataLen(right)); - } - } else if (pLeft->numOfRows == 1) { - for (; i >= 0 && i < pRight->numOfRows; i += step, output += len) { - char *right = POINTER_SHIFT(pRight->data, pRight->bytes * i); - if (isNull(pLeft->data, pLeftCol->info.type) || isNull(right, pRight->info.type)) { - setVardataNull(output, TSDB_DATA_TYPE_BINARY); - continue; - } - - memcpy(varDataVal(output), varDataVal(pLeft->data), varDataLen(pLeft->data)); - memcpy(varDataVal(output) + varDataLen(pLeft->data), varDataVal(right), varDataLen(right)); - varDataSetLen(output, varDataLen(pLeft->data) + varDataLen(right)); - } - } else if (pRight->numOfRows == 1) { - for (; i >= 0 && i < pLeft->numOfRows; i += step, output += len) { - char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); - if (isNull(left, pLeftCol->info.type) || isNull(pRight->data, pRight->info.type)) { - SET_DOUBLE_NULL(output); - continue; - } - - memcpy(varDataVal(output), varDataVal(left), varDataLen(pRight->data)); - memcpy(varDataVal(output) + varDataLen(left), varDataVal(pRight->data), varDataLen(pRight->data)); - varDataSetLen(output, varDataLen(left) + varDataLen(pRight->data)); - } - } -#endif -} - static void vectorBitAndHelper(SColumnInfoData *pLeftCol, SColumnInfoData *pRightCol, SColumnInfoData *pOutputCol, int32_t numOfRows, int32_t step, int32_t i) { _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 02b74c260b..08defcd671 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -253,6 +253,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) { if (pTask == NULL) { tdbFree(pKey); tdbFree(pVal); + tdbTbcClose(pCur); return -1; } tDecoderInit(&decoder, (uint8_t*)pVal, vLen); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index c3c58e125a..4b3affa9de 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -526,7 +526,7 @@ int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { return tdbTbDelete(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), &pState->txn); } -SStreamStateCur* streamStateSessionSeekKeyPrev(SStreamState* pState, const SSessionKey* key) { +SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { return NULL; @@ -544,7 +544,7 @@ SStreamStateCur* streamStateSessionSeekKeyPrev(SStreamState* pState, const SSess streamStateFreeCur(pCur); return NULL; } - if (c > 0) return pCur; + if (c >= 0) return pCur; if (tdbTbcMoveToPrev(pCur->pCur) < 0) { streamStateFreeCur(pCur); @@ -572,7 +572,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess streamStateFreeCur(pCur); return NULL; } - if (c > 0) return pCur; + if (c < 0) return pCur; if (tdbTbcMoveToNext(pCur->pCur) < 0) { streamStateFreeCur(pCur); @@ -630,7 +630,7 @@ SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKe streamStateCurPrev(pState, pCur); SSessionKey tmpKey = *key; int32_t code = streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); - if (code == TSDB_CODE_SUCCESS && sessionKeyCmpr(key, &tmpKey) == 0) { + if (code == 0 && sessionKeyCmpr(key, &tmpKey) == 0) { resKey = tmpKey; } else { break; @@ -640,9 +640,28 @@ SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKe return streamStateSessionGetRanomCur(pState, &resKey); } +int32_t streamStateSessionGetKey(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { + SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + SSessionKey resKey = *key; + int32_t res = -1; + while (1) { + SSessionKey tmpKey = *key; + int32_t code = streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); + if (code == 0 && sessionKeyCmpr(key, &tmpKey) == 0) { + res = 0; + resKey = tmpKey; + } else { + break; + } + streamStateCurPrev(pState, pCur); + } + *curKey = resKey; + return res; +} + int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { // todo refactor - SStreamStateCur* pCur = streamStateSessionGetCur(pState, key); + SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); int32_t size = *pVLen; void* tmp = NULL; *pVal = tdbRealloc(NULL, size); @@ -659,7 +678,7 @@ int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { // todo refactor - int32_t res = TSDB_CODE_SUCCESS; + int32_t res = 0; SSessionKey tmpKey = *key; int32_t valSize = *pVLen; void* tmp = tdbRealloc(NULL, valSize); @@ -667,21 +686,14 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch return -1; } - SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); int32_t code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); - if (code == TSDB_CODE_SUCCESS) { - memcpy(tmp, *pVal, valSize); - *pVal = tmp; - streamStateFreeCur(pCur); - return res; - } - streamStateFreeCur(pCur); + if (code == 0) { + if (key->win.skey <= tmpKey.win.skey && tmpKey.win.ekey <= key->win.ekey) { + memcpy(tmp, *pVal, valSize); + goto _end; + } - streamStateSessionPut(pState, key, NULL, 0); - pCur = streamStateSessionGetRanomCur(pState, key); - streamStateCurPrev(pState, pCur); - code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); - if (code == TSDB_CODE_SUCCESS) { void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); @@ -689,11 +701,9 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch } } - streamStateFreeCur(pCur); - *key = tmpKey; - pCur = streamStateSessionSeekKeyNext(pState, key); + streamStateCurNext(pState, pCur); code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); - if (code == TSDB_CODE_SUCCESS) { + if (code == 0) { void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); @@ -708,7 +718,6 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch _end: *pVal = tmp; - streamStateSessionDel(pState, &tmpKey); streamStateFreeCur(pCur); return res; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 73cc34a86f..8b029b06d6 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -53,6 +53,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF if (strcmp(TDB_MAINDB_NAME, tbname)) { pPager = tdbEnvGetPager(pEnv, fFullName); if (!pPager) { + tdbOsFree(pTb); return -1; } @@ -72,6 +73,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF if (pPager == NULL) { ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); if (ret < 0) { + tdbOsFree(pTb); return -1; } @@ -94,6 +96,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname); ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); if (ret < 0) { + tdbOsFree(pTb); return -1; } diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index 6ead7c0dd6..305e91f62c 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -119,13 +119,13 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in return cret; } -TEST(TdbOVFLPagesTest, DISABLED_TbUpsertTest) { - // TEST(TdbOVFLPagesTest, TbUpsertTest) { -} +// TEST(TdbOVFLPagesTest, DISABLED_TbUpsertTest) { +// TEST(TdbOVFLPagesTest, TbUpsertTest) { +//} -TEST(TdbOVFLPagesTest, DISABLED_TbPGetTest) { - // TEST(TdbOVFLPagesTest, TbPGetTest) { -} +// TEST(TdbOVFLPagesTest, DISABLED_TbPGetTest) { +// TEST(TdbOVFLPagesTest, TbPGetTest) { +//} static void generateBigVal(char *val, int valLen) { for (int i = 0; i < valLen; ++i) { @@ -193,10 +193,8 @@ static void insertOfp(void) { tdbTxnClose(&txn); } -TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) { - // TEST(TdbOVFLPagesTest, TbInsertTest) { - insertOfp(); -} +// TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) { +TEST(TdbOVFLPagesTest, TbInsertTest) { insertOfp(); } // TEST(TdbOVFLPagesTest, DISABLED_TbGetTest) { TEST(TdbOVFLPagesTest, TbGetTest) { @@ -422,13 +420,13 @@ TEST(tdb_test, simple_insert1) { for (int i = 1; i <= nData; i++) { sprintf(key, "key%d", i); - sprintf(val, "value%d", i); + // sprintf(val, "value%d", i); ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); ASSERT(ret == 0); GTEST_ASSERT_EQ(ret, 0); - GTEST_ASSERT_EQ(vLen, strlen(val)); + GTEST_ASSERT_EQ(vLen, sizeof(val) / sizeof(val[0])); GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3616b4808f..cc381d06a7 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -811,7 +811,7 @@ static bool addHandleToAcceptloop(void* arg) { tError("failed to bind:%s", uv_err_name(err)); return false; } - if ((err = uv_listen((uv_stream_t*)&srv->server, 512, uvOnAcceptCb)) != 0) { + if ((err = uv_listen((uv_stream_t*)&srv->server, 4096 * 2, uvOnAcceptCb)) != 0) { tError("failed to listen:%s", uv_err_name(err)); terrno = TSDB_CODE_RPC_PORT_EADDRINUSE; return false; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 60c8591823..8f493ddd85 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -278,12 +278,12 @@ static int32_t walFetchBodyNew(SWalReader *pRead) { wDebug("vgId:%d, wal starts to fetch body, index:%" PRId64, pRead->pWal->cfg.vgId, ver); if (pRead->capacity < pReadHead->bodyLen) { - void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); + SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } - pRead->pHead = (SWalCkHead *)ptr; + pRead->pHead = ptr; pReadHead = &pRead->pHead->head; pRead->capacity = pReadHead->bodyLen; } @@ -398,12 +398,12 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { int64_t ver = pReadHead->version; if (pRead->capacity < pReadHead->bodyLen) { - void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalCkHead) + pReadHead->bodyLen); + SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(*ppHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } - *ppHead = (SWalCkHead *)ptr; + *ppHead = ptr; pReadHead = &((*ppHead)->head); pRead->capacity = pReadHead->bodyLen; } @@ -493,13 +493,14 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { } if (pReader->capacity < pReader->pHead->head.bodyLen) { - void *ptr = taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReader->pHead->head.bodyLen); + SWalCkHead *ptr = + (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReader->pHead->head.bodyLen); if (ptr == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; taosThreadMutexUnlock(&pReader->mutex); return -1; } - pReader->pHead = (SWalCkHead *)ptr; + pReader->pHead = ptr; pReader->capacity = pReader->pHead->head.bodyLen; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index edb2f0380e..0bf9e7cc33 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -944,6 +944,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { if (taosReadFile(pFile, buf, fileSize) <= 0) { taosCloseFile(&pFile); uError("load json file error: %s", filepath); + taosMemoryFreeClear(buf); return -1; } taosCloseFile(&pFile); @@ -953,6 +954,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { if (jsonParseError != NULL) { uError("load json file parse error: %s", jsonParseError); } + taosMemoryFreeClear(buf); return -1; } taosMemoryFreeClear(buf); diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index eb62e12a19..8748f8df3f 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -141,12 +141,14 @@ int32_t taosQueueItemSize(STaosQueue *queue) { } int64_t taosQueueMemorySize(STaosQueue *queue) { - if (queue == NULL) return 0; - +#if 1 + return queue->memOfItems; +#else taosThreadMutexLock(&queue->mutex); int64_t memOfItems = queue->memOfItems; taosThreadMutexUnlock(&queue->mutex); return memOfItems; +#endif } void *taosAllocateQitem(int32_t size, EQItype itype) { @@ -298,9 +300,6 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { return num; } -void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } -int32_t taosQallItemSize(STaosQall *qall) { return qall->numOfItems; } - STaosQset *taosOpenQset() { STaosQset *qset = taosMemoryCalloc(sizeof(STaosQset), 1); if (qset == NULL) { @@ -405,8 +404,6 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { uDebug("queue:%p is removed from qset:%p", queue, qset); } -int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; } - int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo) { STaosQnode *pNode = NULL; int32_t code = 0; @@ -497,6 +494,12 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo * return code; } +int32_t taosQallItemSize(STaosQall *qall) { return qall->numOfItems; } +void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } +int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; } + +#if 0 + void taosResetQsetThread(STaosQset *qset, void *pItem) { if (pItem == NULL) return; STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); @@ -507,3 +510,5 @@ void taosResetQsetThread(STaosQset *qset, void *pItem) { } taosThreadMutexUnlock(&qset->mutex); } + +#endif diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index e353546ca7..f7d4173d3f 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -28,10 +28,7 @@ int32_t tQWorkerInit(SQWorkerPool *pool) { return -1; } - if (taosThreadMutexInit(&pool->mutex, NULL)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + (void)taosThreadMutexInit(&pool->mutex, NULL); for (int32_t i = 0; i < pool->max; ++i) { SQWorker *worker = pool->workers + i; @@ -97,14 +94,10 @@ static void *tQWorkerThreadFp(SQWorker *worker) { } STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { - taosThreadMutexLock(&pool->mutex); STaosQueue *queue = taosOpenQueue(); - if (queue == NULL) { - taosThreadMutexUnlock(&pool->mutex); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } + if (queue == NULL) return NULL; + taosThreadMutexLock(&pool->mutex); taosSetQueueFp(queue, fp, NULL); taosAddIntoQset(pool->qset, queue, ahandle); @@ -118,7 +111,6 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tQWorkerThreadFp, worker) != 0) { - uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosCloseQueue(queue); terrno = TSDB_CODE_OUT_OF_MEMORY; queue = NULL; @@ -150,10 +142,7 @@ int32_t tWWorkerInit(SWWorkerPool *pool) { return -1; } - if (taosThreadMutexInit(&pool->mutex, NULL) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + (void)taosThreadMutexInit(&pool->mutex, NULL); for (int32_t i = 0; i < pool->max; ++i) { SWWorker *worker = pool->workers + i; @@ -225,48 +214,27 @@ static void *tWWorkerThreadFp(SWWorker *worker) { STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { taosThreadMutexLock(&pool->mutex); SWWorker *worker = pool->workers + pool->nextId; + int32_t code = -1; STaosQueue *queue = taosOpenQueue(); - if (queue == NULL) { - taosThreadMutexUnlock(&pool->mutex); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } + if (queue == NULL) goto _OVER; taosSetQueueFp(queue, NULL, fp); - if (worker->qset == NULL) { worker->qset = taosOpenQset(); - if (worker->qset == NULL) { - taosCloseQueue(queue); - taosThreadMutexUnlock(&pool->mutex); - return NULL; - } + if (worker->qset == NULL) goto _OVER; taosAddIntoQset(worker->qset, queue, ahandle); worker->qall = taosAllocateQall(); - if (worker->qall == NULL) { - taosCloseQset(worker->qset); - taosCloseQueue(queue); - taosThreadMutexUnlock(&pool->mutex); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } + if (worker->qall == NULL) goto _OVER; + TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) goto _OVER; - if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) { - uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); - taosFreeQall(worker->qall); - taosCloseQset(worker->qset); - taosCloseQueue(queue); - terrno = TSDB_CODE_OUT_OF_MEMORY; - queue = NULL; - } else { - uDebug("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); - pool->nextId = (pool->nextId + 1) % pool->max; - } + uDebug("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); + pool->nextId = (pool->nextId + 1) % pool->max; taosThreadAttrDestroy(&thAttr); pool->num++; @@ -276,10 +244,20 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pool->nextId = (pool->nextId + 1) % pool->max; } - taosThreadMutexUnlock(&pool->mutex); uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle); + code = 0; - return queue; +_OVER: + taosThreadMutexUnlock(&pool->mutex); + + if (code == -1) { + if (queue != NULL) taosCloseQueue(queue); + if (worker->qset != NULL) taosCloseQset(worker->qset); + if (worker->qall != NULL) taosFreeQall(worker->qall); + return NULL; + } else { + return queue; + } } void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) { @@ -292,15 +270,11 @@ int32_t tSingleWorkerInit(SSingleWorker *pWorker, const SSingleWorkerCfg *pCfg) pPool->name = pCfg->name; pPool->min = pCfg->min; pPool->max = pCfg->max; - if (tQWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + if (tQWorkerInit(pPool) != 0) return -1; + pWorker->queue = tQWorkerAllocQueue(pPool, pCfg->param, pCfg->fp); - if (pWorker->queue == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + if (pWorker->queue == NULL) return -1; + pWorker->name = pCfg->name; return 0; } @@ -320,15 +294,11 @@ int32_t tMultiWorkerInit(SMultiWorker *pWorker, const SMultiWorkerCfg *pCfg) { SWWorkerPool *pPool = &pWorker->pool; pPool->name = pCfg->name; pPool->max = pCfg->max; - if (tWWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + if (tWWorkerInit(pPool) != 0) return -1; + pWorker->queue = tWWorkerAllocQueue(pPool, pCfg->param, pCfg->fp); - if (pWorker->queue == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + if (pWorker->queue == NULL) return -1; + pWorker->name = pCfg->name; return 0; } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index fc139a1533..e07ec54182 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -228,7 +228,7 @@ ./test.sh -f tsim/table/vgroup.sim # ---- stream -./test.sh -f tsim/stream/basic0.sim +./test.sh -f tsim/stream/basic0.sim -v ./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/drop_stream.sim diff --git a/tests/script/tsim/insert/basic.sim b/tests/script/tsim/insert/basic.sim index c926cbc8b0..ec8a61bb04 100644 --- a/tests/script/tsim/insert/basic.sim +++ b/tests/script/tsim/insert/basic.sim @@ -12,14 +12,14 @@ $tb = $tbPrefix . $i print =============== step1 sql drop database -x step1 step1: -sql create database $db vgroups 2 +sql create database $db vgroups 2 precision 'ns' sql use $db sql create table $tb (ts timestamp, speed int) $x = 0 -while $x < 10 +while $x < 110 $cc = $x * 60000 - $ms = 1601481600000 + $cc + $ms = 1601481600000000000 + $cc sql insert into $tb values ($ms , $x ) $x = $x + 1 @@ -27,9 +27,9 @@ endw print =============== step 2 $x = 0 -while $x < 5 +while $x < 110 $cc = $x * 60000 - $ms = 1551481600000 + $cc + $ms = 1551481600000000000 + $cc sql insert into $tb values ($ms , $x ) $x = $x + 1 @@ -38,8 +38,29 @@ endw sql select * from $tb print $rows points data are retrieved -if $rows != 15 then +if $rows != 220 then return -1 endi +# error +print $data1 +print $data[1000][1] +print $data[1][1000] +print $data[1000][1000] +$a1 = 0 +$a2 = 0 +$a3 = $a1 % $a2 +print $a3 + +$val = \\\1 +print ====> $val + +sql_slow select * from $tb + +sql close +sql connect +sql close +sql connect root + + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/basic0.sim b/tests/script/tsim/stream/basic0.sim index 6d05f69dcf..8e3e99978b 100644 --- a/tests/script/tsim/stream/basic0.sim +++ b/tests/script/tsim/stream/basic0.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c debugflag -v 131 -system sh/exec.sh -n dnode1 -s start -v +system sh/exec.sh -n dnode1 -s start sql connect print =============== create database diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 1d733191b7..96e057c9f8 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -94,7 +94,7 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): tdLog.exit("vgroup_id is null!") if "database_name" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["database_name"]) < 0: tdLog.exit("database_name is null!") - if "tables_num" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["tables_num"]!= 0: + if "tables_num" not in infoDict["vgroup_infos"][index]: tdLog.exit("tables_num is null!") if "status" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["status"]) < 0 : tdLog.exit("status is null!") @@ -294,6 +294,10 @@ class TDTestCase: vgroups = "30" sql = "create database db3 vgroups " + vgroups tdSql.query(sql) + sql = "create table db3.stb (ts timestamp, f int) tags (t int)" + tdSql.query(sql) + sql = "create table db3.tb using db3.stb tags (1)" + tdSql.query(sql) # create http server: bing ip/port , and request processor if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 46eaf6cff1..8d980ba653 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -562,23 +562,15 @@ void parseCommand(SWords* command, bool pattern) { // free SShellCmd void freeCommand(SWords* command) { - SWord* word = command->head; - if (word == NULL) { - return; - } - + SWord* item = command->head; // loop - while (word->next) { - SWord* tmp = word; - word = word->next; + while (item) { + SWord* tmp = item; + item = item->next; // if malloc need free if (tmp->free && tmp->word) taosMemoryFree(tmp->word); taosMemoryFree(tmp); } - - // if malloc need free - if (word->free && word->word) taosMemoryFree(word->word); - taosMemoryFree(word); } void GenerateVarType(int type, char** p, int count) { @@ -1204,11 +1196,11 @@ bool nextMatchCommand(TAOS* con, SShellCmd* cmd, SWords* firstMatch) { #endif // free + freeCommand(input); if (input->source) { taosMemoryFree(input->source); input->source = NULL; } - freeCommand(input); taosMemoryFree(input); return true; @@ -1377,7 +1369,7 @@ bool appendAfterSelect(TAOS* con, SShellCmd* cmd, char* sql, int32_t len) { bool ret = false; if (from == NULL) { bool fieldEnd = fieldsInputEnd(p); - // cheeck fields input end then insert from keyword + // check fields input end then insert from keyword if (fieldEnd && p[len - 1] == ' ') { shellInsertChar(cmd, "from", 4); taosMemoryFree(p); diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index c34b22b691..047aeb5b95 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -101,8 +101,13 @@ void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { /* update the values */ cmd->commandSize += size; cmd->cursorOffset += size; - cmd->screenOffset += taosWcharWidth(wc); - cmd->endOffset += taosWcharWidth(wc); + for (int i = 0; i < size; i++) { + taosMbToWchar(&wc, c + i, size); + cmd->screenOffset += taosWcharWidth(wc); + cmd->endOffset += taosWcharWidth(wc); + } + // set string end + cmd->command[cmd->commandSize] = 0; #ifdef WINDOWS #else shellShowOnScreen(cmd); @@ -123,6 +128,8 @@ void shellBackspaceChar(SShellCmd *cmd) { cmd->cursorOffset -= size; cmd->screenOffset -= width; cmd->endOffset -= width; + // set string end + cmd->command[cmd->commandSize] = 0; shellShowOnScreen(cmd); } } @@ -136,6 +143,8 @@ void shellClearLineBefore(SShellCmd *cmd) { cmd->cursorOffset = 0; cmd->screenOffset = 0; cmd->endOffset = cmd->commandSize; + // set string end + cmd->command[cmd->commandSize] = 0; shellShowOnScreen(cmd); } @@ -160,6 +169,8 @@ void shellDeleteChar(SShellCmd *cmd) { cmd->commandSize - cmd->cursorOffset - size); cmd->commandSize -= size; cmd->endOffset -= width; + // set string end + cmd->command[cmd->commandSize] = 0; shellShowOnScreen(cmd); } } diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index 346757b76f..0628570904 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -309,27 +309,24 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) { } SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) { - if (match == NULL) { - match = (SMatch*)taosMemoryMalloc(sizeof(SMatch)); - memset(match, 0, sizeof(SMatch)); + SMatch* rMatch = match; // define return match + if (rMatch == NULL) { + rMatch = (SMatch*)taosMemoryMalloc(sizeof(SMatch)); + memset(rMatch, 0, sizeof(SMatch)); } switch (tire->type) { case TIRE_TREE: - matchPrefixFromTree(tire, prefix, match); + matchPrefixFromTree(tire, prefix, rMatch); + break; case TIRE_LIST: - matchPrefixFromList(tire, prefix, match); + matchPrefixFromList(tire, prefix, rMatch); + break; default: break; } - // return if need - if (match->count == 0) { - freeMatch(match); - match = NULL; - } - - return match; + return rMatch; } // get all items from tires tree @@ -378,8 +375,10 @@ SMatch* enumAll(STire* tire) { switch (tire->type) { case TIRE_TREE: enumFromTree(tire, match); + break; case TIRE_LIST: enumFromList(tire, match); + break; default: break; } diff --git a/utils/test/c/createTable.c b/utils/test/c/createTable.c index 783ed85adc..9a39fdd975 100644 --- a/utils/test/c/createTable.c +++ b/utils/test/c/createTable.c @@ -330,11 +330,11 @@ void parseArgument(int32_t argc, char *argv[]) { printHelp(); exit(0); } else if (strcmp(argv[i], "-d") == 0) { - strcpy(dbName, argv[++i]); + tstrncpy(dbName, argv[++i], sizeof(dbName)); } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); + tstrncpy(configDir, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-s") == 0) { - strcpy(stbName, argv[++i]); + tstrncpy(stbName, argv[++i], sizeof(stbName)); } else if (strcmp(argv[i], "-t") == 0) { numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { diff --git a/utils/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h index 9438e11fbd..f512b119b4 100644 --- a/utils/tsim/inc/simInt.h +++ b/utils/tsim/inc/simInt.h @@ -19,20 +19,20 @@ #include "os.h" #include "cJSON.h" -#include "tconfig.h" #include "taos.h" #include "taoserror.h" +#include "tconfig.h" +#include "tglobal.h" #include "tidpool.h" #include "tlog.h" #include "ttimer.h" #include "ttypes.h" #include "tutil.h" -#include "tglobal.h" #define MAX_MAIN_SCRIPT_NUM 10 #define MAX_BACKGROUND_SCRIPT_NUM 10 #define MAX_FILE_NAME_LEN 256 -#define MAX_ERROR_LEN 1024 +#define MAX_ERROR_LEN 4096 #define MAX_QUERY_VALUE_LEN 1024 #define MAX_QUERY_COL_NUM 100 #define MAX_QUERY_ROW_NUM 100 @@ -55,12 +55,42 @@ #define FAILED_POSTFIX "" #endif -#define simFatal(...) { if (simDebugFlag & DEBUG_FATAL) { taosPrintLog("SIM FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} -#define simError(...) { if (simDebugFlag & DEBUG_ERROR) { taosPrintLog("SIM ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} -#define simWarn(...) { if (simDebugFlag & DEBUG_WARN) { taosPrintLog("SIM WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define simInfo(...) { if (simDebugFlag & DEBUG_INFO) { taosPrintLog("SIM ", DEBUG_INFO, 255, __VA_ARGS__); }} -#define simDebug(...) { if (simDebugFlag & DEBUG_DEBUG) { taosPrintLog("SIM ", DEBUG_DEBUG, simDebugFlag, __VA_ARGS__); }} -#define simTrace(...) { if (simDebugFlag & DEBUG_TRACE) { taosPrintLog("SIM ", DEBUG_TRACE, simDebugFlag, __VA_ARGS__); }} +#define simFatal(...) \ + { \ + if (simDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("SIM FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ + } +#define simError(...) \ + { \ + if (simDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("SIM ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ + } +#define simWarn(...) \ + { \ + if (simDebugFlag & DEBUG_WARN) { \ + taosPrintLog("SIM WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ + } +#define simInfo(...) \ + { \ + if (simDebugFlag & DEBUG_INFO) { \ + taosPrintLog("SIM ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ + } +#define simDebug(...) \ + { \ + if (simDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("SIM ", DEBUG_DEBUG, simDebugFlag, __VA_ARGS__); \ + } \ + } +#define simTrace(...) \ + { \ + if (simDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("SIM ", DEBUG_TRACE, simDebugFlag, __VA_ARGS__); \ + } \ + } enum { SIM_SCRIPT_TYPE_MAIN, SIM_SCRIPT_TYPE_BACKGROUND }; @@ -143,7 +173,7 @@ typedef struct _script_t { char *optionBuffer; SCmdLine *lines; // command list SVariable variables[MAX_VAR_LEN]; - TdThread bgPid; + TdThread bgPid; char auth[128]; struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; } SScript; @@ -182,4 +212,4 @@ bool simExecuteLineInsertCmd(SScript *script, char *option); bool simExecuteLineInsertErrorCmd(SScript *script, char *option); void simVisuallizeOption(SScript *script, char *src, char *dst); -#endif /*_TD_SIM_INT_H_*/ \ No newline at end of file +#endif /*_TD_SIM_INT_H_*/ diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 7218ea39f8..258c611557 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -21,35 +21,20 @@ void simLogSql(char *sql, bool useSharp) { char filename[256]; sprintf(filename, "%s/sim.sql", simScriptDir); if (pFile == NULL) { - // fp = fopen(filename, "w"); pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - if (pFile == NULL) { - fprintf(stderr, "ERROR: failed to open file: %s\n", filename); - return; + } + + if (pFile != NULL) { + if (useSharp) { + taosFprintfFile(pFile, "# %s;\n", sql); + } else { + taosFprintfFile(pFile, "%s;\n", sql); } - } - if (useSharp) { - taosFprintfFile(pFile, "# %s;\n", sql); - } else { - taosFprintfFile(pFile, "%s;\n", sql); - } - taosFsyncFile(pFile); + taosFsyncFile(pFile); + } } -#if 0 -char *simParseArbitratorName(char *varName) { - static char hostName[140]; -#ifdef WINDOWS - taosGetFqdn(hostName); - sprintf(&hostName[strlen(hostName)], ":%d", 8000); -#else - sprintf(hostName, "%s:%d", "localhost", 8000); -#endif - return hostName; -} -#endif - char *simParseHostName(char *varName) { static char hostName[140]; #ifdef WINDOWS @@ -65,18 +50,18 @@ char *simParseHostName(char *varName) { } static void simFindFirstNum(const char *begin, int32_t beginLen, int32_t *num) { - if (beginLen <= 5) { - *num = 0; - } else { + *num = 0; + + if (beginLen > 5) { *num = atoi(begin + 5); } } static void simFindSecondNum(const char *begin, int32_t beginLen, int32_t *num) { + *num = 0; + const char *number = strstr(begin, "]["); - if (number == NULL) { - *num = 0; - } else { + if (number != NULL) { *num = atoi(number + 2); } } @@ -91,10 +76,10 @@ static void simFindFirstKeyVal(const char *begin, int32_t beginLen, char *key, i } static void simFindSecondKeyNum(const char *begin, int32_t beginLen, int32_t *num) { + *num = 0; + const char *number = strstr(begin, ")["); - if (number == NULL) { - *num = 0; - } else { + if (number != NULL) { *num = atoi(number + 2); } } @@ -104,12 +89,6 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { return simParseHostName(varName); } -#if 0 - if (strncmp(varName, "arbitrator", 10) == 0) { - return simParseArbitratorName(varName); - } -#endif - if (strncmp(varName, "error", varLen) == 0) return script->error; if (strncmp(varName, "rows", varLen) == 0) return script->rows; @@ -153,9 +132,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { } else if (varName[5] == '_') { // data2_db int32_t col = varName[4] - '0'; - if (col < 0 || col >= MAX_QUERY_COL_NUM) { - return "null"; - } + col = col % MAX_QUERY_COL_NUM; char *keyName; int32_t keyLen; @@ -171,9 +148,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { } else if (varName[6] == '_') { // data21_db int32_t col = (varName[4] - '0') * 10 + (varName[5] - '0'); - if (col < 0 || col >= MAX_QUERY_COL_NUM) { - return "null"; - } + col = col % MAX_QUERY_COL_NUM; char *keyName; int32_t keyLen; @@ -185,17 +160,12 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { return script->data[i][col]; } } - return "null"; } else { // $data00 int32_t row = varName[4] - '0'; int32_t col = varName[5] - '0'; - if (row < 0 || row >= MAX_QUERY_ROW_NUM) { - return "null"; - } - if (col < 0 || col >= MAX_QUERY_COL_NUM) { - return "null"; - } + row = row % MAX_QUERY_ROW_NUM; + col = col % MAX_QUERY_COL_NUM; simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]); return script->data[row][col]; @@ -253,9 +223,9 @@ int32_t simExecuteExpression(SScript *script, char *exp) { rest = paGetToken(rest, &op2, &op2Len); if (var1[0] == '$') - strcpy(t0, simGetVariable(script, var1 + 1, var1Len - 1)); + tstrncpy(t0, simGetVariable(script, var1 + 1, var1Len - 1), sizeof(t0)); else { - memcpy(t0, var1, var1Len); + tstrncpy(t0, var1, var1Len); t0[var1Len] = 0; } @@ -578,6 +548,8 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { while (1) { var = strchr(src, '$'); if (var == NULL) break; + +#if 0 if (var && ((var - src - 1) > 0) && *(var - 1) == '\\') { srcLen = (int32_t)(var - src - 1); memcpy(dst + dstLen, src, srcLen); @@ -585,6 +557,7 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { src = var; break; } +#endif srcLen = (int32_t)(var - src); memcpy(dst + dstLen, src, srcLen); @@ -657,11 +630,11 @@ bool simCreateTaosdConnect(SScript *script, char *rest) { } bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { - char timeStr[80] = {0}; - time_t tt; - struct tm tp; - SCmdLine *line = &script->lines[script->linePos]; - int32_t ret = -1; + char timeStr[80] = {0}; + time_t tt; + struct tm tp; + SCmdLine *line = &script->lines[script->linePos]; + int32_t ret = -1; TAOS_RES *pSql = NULL; @@ -705,17 +678,6 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { int32_t numOfRows = 0; int32_t num_fields = taos_field_count(pSql); if (num_fields != 0) { - if (pSql == NULL) { - simDebug("script:%s, taos:%p, %s failed, result is null", script->fileName, script->taos, rest); - if (line->errorJump == SQL_JUMP_TRUE) { - script->linePos = line->jump; - return true; - } - - sprintf(script->error, "lineNum:%d. result set null, sql:%s", line->lineNum, rest); - return false; - } - TAOS_ROW row; while ((row = taos_fetch_row(pSql))) { @@ -942,30 +904,6 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { } } - if (strncmp(rest, "connect", 7) == 0) { - if (!simCreateTaosdConnect(script, rest)) { - return false; - } - script->linePos++; - return true; - } - - if (script->taos == NULL) { - if (!simCreateTaosdConnect(script, "connect root")) { - if (line->errorJump == SQL_JUMP_TRUE) { - script->linePos = line->jump; - return true; - } - return false; - } - } - - if (strncmp(rest, "close", 5) == 0) { - simCloseTaosdConnect(script); - script->linePos++; - return true; - } - TAOS_RES *pSql = taos_query(script->taos, rest); int32_t ret = taos_errno(pSql); taos_free_result(pSql); diff --git a/utils/tsim/src/simParse.c b/utils/tsim/src/simParse.c index ae523edfc4..cdd918ac00 100644 --- a/utils/tsim/src/simParse.c +++ b/utils/tsim/src/simParse.c @@ -717,20 +717,12 @@ bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } -#if 0 bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseSqlCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_SQL_SLOW; return true; } -bool simParseRestfulCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - simParseSqlCmd(rest, pCmd, lineNum); - cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; - return true; -} -#endif - bool simParseSystemCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; @@ -1058,7 +1050,6 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteSqlErrorCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); -#if 0 cmdno = SIM_CMD_SQL_SLOW; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "sql_slow"); @@ -1067,15 +1058,6 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); - cmdno = SIM_CMD_RESTFUL; - simCmdList[cmdno].cmdno = cmdno; - strcpy(simCmdList[cmdno].name, "restful"); - simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); - simCmdList[cmdno].parseCmd = simParseRestfulCmd; - simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; - simAddCmdIntoHash(&(simCmdList[cmdno])); -#endif - /* test is only an internal command */ cmdno = SIM_CMD_TEST; simCmdList[cmdno].cmdno = cmdno;