Merge branch '3.0' into fix/3_liaohj

This commit is contained in:
Haojun Liao 2024-07-17 11:25:19 +08:00
commit 53ed030967
128 changed files with 9753 additions and 7295 deletions

View File

@ -60,8 +60,8 @@ typedef struct {
} SCheckpoint; } SCheckpoint;
int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint); int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint);
void cos_cp_close(TdFilePtr fd); int32_t cos_cp_close(TdFilePtr fd);
void cos_cp_remove(char const* filepath); int32_t cos_cp_remove(char const* filepath);
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint); int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
int32_t cos_cp_dump(SCheckpoint* checkpoint); int32_t cos_cp_dump(SCheckpoint* checkpoint);

View File

@ -136,7 +136,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag); int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
void tRowGetPrimaryKey(SRow *pRow, SRowKey *key); void tRowGetPrimaryKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2); int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2);
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc); void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
// SRowIter ================================ // SRowIter ================================
int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);

View File

@ -31,7 +31,7 @@ typedef struct SCorEpSet {
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t len); int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t len);
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp); int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2); bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
void epsetAssign(SEpSet* dst, const SEpSet* pSrc); void epsetAssign(SEpSet* dst, const SEpSet* pSrc);

View File

@ -17,13 +17,13 @@
#define _TD_COMMON_MSG_CB_H_ #define _TD_COMMON_MSG_CB_H_
#include "os.h" #include "os.h"
#include "tmsg.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct SRpcMsg SRpcMsg; typedef struct SRpcMsg SRpcMsg;
typedef struct SEpSet SEpSet;
typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SMgmtWrapper SMgmtWrapper;
typedef struct SRpcHandleInfo SRpcHandleInfo; typedef struct SRpcHandleInfo SRpcHandleInfo;
@ -46,7 +46,7 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg);
typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
typedef void (*SendRspFp)(SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg);
typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg); typedef void (*RegisterBrokenLinkArgFp)(struct SRpcMsg* pMsg);
typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
typedef void (*ReportStartup)(const char* name, const char* desc); typedef void (*ReportStartup)(const char* name, const char* desc);

View File

@ -81,7 +81,7 @@ typedef struct {
// uint64_t uid; // child table uid, may be useful // uint64_t uid; // child table uid, may be useful
} RandTableName; } RandTableName;
void buildChildTableName(RandTableName* rName); int32_t buildChildTableName(RandTableName* rName);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -132,16 +132,16 @@
#define TK_TABLE 114 #define TK_TABLE 114
#define TK_NK_LP 115 #define TK_NK_LP 115
#define TK_NK_RP 116 #define TK_NK_RP 116
#define TK_STABLE 117 #define TK_USING 117
#define TK_COLUMN 118 #define TK_FILE 118
#define TK_MODIFY 119 #define TK_STABLE 119
#define TK_RENAME 120 #define TK_COLUMN 120
#define TK_TAG 121 #define TK_MODIFY 121
#define TK_SET 122 #define TK_RENAME 122
#define TK_NK_EQ 123 #define TK_TAG 123
#define TK_USING 124 #define TK_SET 124
#define TK_TAGS 125 #define TK_NK_EQ 125
#define TK_FILE 126 #define TK_TAGS 126
#define TK_BOOL 127 #define TK_BOOL 127
#define TK_TINYINT 128 #define TK_TINYINT 128
#define TK_SMALLINT 129 #define TK_SMALLINT 129

View File

@ -50,7 +50,7 @@ typedef struct {
int32_t auditInit(const SAuditCfg *pCfg); int32_t auditInit(const SAuditCfg *pCfg);
void auditCleanup(); void auditCleanup();
void auditSend(SJson *pJson); int32_t auditSend(SJson *pJson);
void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,
char *detail, int32_t len); char *detail, int32_t len);
void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,

View File

@ -152,7 +152,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
* @return * @return
*/ */
int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
int32_t* tversion, int32_t idx); int32_t* tversion, int32_t idx, bool* tbGet);
/** /**
* The main task execution function, including query on both table and multiple tables, * The main task execution function, including query on both table and multiple tables,

View File

@ -105,7 +105,7 @@ typedef struct SMTbCursor {
} SMTbCursor; } SMTbCursor;
typedef struct SMCtbCursor { typedef struct SMCtbCursor {
SMeta* pMeta; struct SMeta* pMeta;
void* pCur; void* pCur;
tb_uid_t suid; tb_uid_t suid;
void* pKey; void* pKey;
@ -134,7 +134,7 @@ typedef struct SMetaTableInfo {
} SMetaTableInfo; } SMetaTableInfo;
typedef struct SSnapContext { typedef struct SSnapContext {
SMeta* pMeta; struct SMeta* pMeta;
int64_t snapVersion; int64_t snapVersion;
void* pCur; void* pCur;
int64_t suid; int64_t suid;
@ -178,7 +178,7 @@ typedef struct TsdReader {
int32_t (*tsdNextDataBlock)(); int32_t (*tsdNextDataBlock)();
int32_t (*tsdReaderRetrieveBlockSMAInfo)(); int32_t (*tsdReaderRetrieveBlockSMAInfo)();
SSDataBlock *(*tsdReaderRetrieveDataBlock)(); int32_t (*tsdReaderRetrieveDataBlock)();
void (*tsdReaderReleaseDataBlock)(); void (*tsdReaderReleaseDataBlock)();
@ -381,7 +381,7 @@ typedef struct SStateStore {
void** ppVal, int32_t* pVLen); void** ppVal, int32_t* pVLen);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
@ -389,12 +389,12 @@ typedef struct SStateStore {
void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*updateInfoDestroy)(SUpdateInfo* pInfo);
void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo); int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);

View File

@ -221,9 +221,6 @@ typedef struct SCreateSubTableFromFileClause {
bool ignoreExists; bool ignoreExists;
SNodeList* pSpecificTags; SNodeList* pSpecificTags;
char filePath[PATH_MAX]; char filePath[PATH_MAX];
TdFilePtr fp;
SArray* aCreateTbData;
SArray* aTagIndexs;
} SCreateSubTableFromFileClause; } SCreateSubTableFromFileClause;
typedef struct SCreateMultiTablesStmt { typedef struct SCreateMultiTablesStmt {

View File

@ -233,7 +233,7 @@ typedef struct SViewNode {
#define IS_WINDOW_JOIN(_stype) ((_stype) == JOIN_STYPE_WIN) #define IS_WINDOW_JOIN(_stype) ((_stype) == JOIN_STYPE_WIN)
#define IS_ASOF_JOIN(_stype) ((_stype) == JOIN_STYPE_ASOF) #define IS_ASOF_JOIN(_stype) ((_stype) == JOIN_STYPE_ASOF)
typedef enum EJoinType { typedef enum EJoinType {
JOIN_TYPE_INNER = 0, JOIN_TYPE_INNER = 0,
JOIN_TYPE_LEFT, JOIN_TYPE_LEFT,
JOIN_TYPE_RIGHT, JOIN_TYPE_RIGHT,
@ -251,7 +251,7 @@ typedef enum EJoinSubType {
JOIN_STYPE_MAX_VALUE JOIN_STYPE_MAX_VALUE
} EJoinSubType; } EJoinSubType;
typedef enum EJoinAlgorithm { typedef enum EJoinAlgorithm {
JOIN_ALGO_UNKNOWN = 0, JOIN_ALGO_UNKNOWN = 0,
JOIN_ALGO_MERGE, JOIN_ALGO_MERGE,
JOIN_ALGO_HASH, JOIN_ALGO_HASH,
@ -454,7 +454,7 @@ typedef struct SSetOperator {
SNode* pLimit; SNode* pLimit;
char stmtName[TSDB_TABLE_NAME_LEN]; char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision; uint8_t precision;
ETimeLineMode timeLineResMode; ETimeLineMode timeLineResMode;
bool timeLineFromOrderBy; bool timeLineFromOrderBy;
bool joinContains; bool joinContains;
} SSetOperator; } SSetOperator;
@ -504,6 +504,10 @@ typedef void (*FFreeTableBlockHash)(SHashObj*);
typedef void (*FFreeVgourpBlockArray)(SArray*); typedef void (*FFreeVgourpBlockArray)(SArray*);
struct SStbRowsDataContext; struct SStbRowsDataContext;
typedef void (*FFreeStbRowsDataContext)(struct SStbRowsDataContext*); typedef void (*FFreeStbRowsDataContext)(struct SStbRowsDataContext*);
struct SCreateTbInfo;
struct SParseFileContext;
typedef void (*FDestroyParseFileContext)(struct SParseFileContext**);
typedef struct SVnodeModifyOpStmt { typedef struct SVnodeModifyOpStmt {
ENodeType nodeType; ENodeType nodeType;
ENodeType sqlNodeType; ENodeType sqlNodeType;
@ -524,7 +528,7 @@ typedef struct SVnodeModifyOpStmt {
SHashObj* pTableNameHashObj; // set of table names for refreshing meta, sync mode SHashObj* pTableNameHashObj; // set of table names for refreshing meta, sync mode
SHashObj* pDbFNameHashObj; // set of db names for refreshing meta, sync mode SHashObj* pDbFNameHashObj; // set of db names for refreshing meta, sync mode
SHashObj* pTableCxtHashObj; // temp SHashObj<tuid, STableDataCxt*> for single request SHashObj* pTableCxtHashObj; // temp SHashObj<tuid, STableDataCxt*> for single request
SArray* pVgDataBlocks; // SArray<SVgroupDataCxt*> SArray* pVgDataBlocks; // SArray<SVgroupDataCxt*>
SVCreateTbReq* pCreateTblReq; SVCreateTbReq* pCreateTblReq;
TdFilePtr fp; TdFilePtr fp;
FFreeTableBlockHash freeHashFunc; FFreeTableBlockHash freeHashFunc;
@ -532,9 +536,13 @@ typedef struct SVnodeModifyOpStmt {
bool usingTableProcessing; bool usingTableProcessing;
bool fileProcessing; bool fileProcessing;
bool stbSyntax; bool stbSyntax;
struct SStbRowsDataContext* pStbRowsCxt; struct SStbRowsDataContext* pStbRowsCxt;
FFreeStbRowsDataContext freeStbRowsCxtFunc; FFreeStbRowsDataContext freeStbRowsCxtFunc;
struct SCreateTbInfo* pCreateTbInfo;
struct SParseFileContext* pParFileCxt;
FDestroyParseFileContext destroyParseFileCxt;
} SVnodeModifyOpStmt; } SVnodeModifyOpStmt;
typedef struct SExplainOptions { typedef struct SExplainOptions {
@ -602,7 +610,7 @@ typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, C
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols); SNodeList** pCols);
int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias, ECollectColType type, int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias, ECollectColType type,
SNodeList** pCols); SNodeList** pCols);
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols); int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols);
typedef bool (*FFuncClassifier)(int32_t funcId); typedef bool (*FFuncClassifier)(int32_t funcId);
@ -618,6 +626,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp);
bool nodesIsComparisonOp(const SOperatorNode* pOp); bool nodesIsComparisonOp(const SOperatorNode* pOp);
bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsJsonOp(const SOperatorNode* pOp);
bool nodesIsRegularOp(const SOperatorNode* pOp); bool nodesIsRegularOp(const SOperatorNode* pOp);
bool nodesIsMatchRegularOp(const SOperatorNode* pOp);
bool nodesIsBitwiseOp(const SOperatorNode* pOp); bool nodesIsBitwiseOp(const SOperatorNode* pOp);
bool nodesExprHasColumn(SNode* pNode); bool nodesExprHasColumn(SNode* pNode);

View File

@ -149,10 +149,10 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
void qDestroyBoundColInfo(void* pInfo); void qDestroyBoundColInfo(void* pInfo);
SQuery* smlInitHandle(); int32_t smlInitHandle(SQuery** query);
int32_t smlBuildRow(STableDataCxt* pTableCxt); int32_t smlBuildRow(STableDataCxt* pTableCxt);
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index);
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt);
void clearColValArraySml(SArray* pCols); void clearColValArraySml(SArray* pCols);
int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,

View File

@ -23,8 +23,6 @@ extern "C" {
#include "catalog.h" #include "catalog.h"
#include "planner.h" #include "planner.h"
extern tsem_t schdRspSem;
typedef struct SQueryProfileSummary { typedef struct SQueryProfileSummary {
int64_t startTs; // Object created and added into the message queue int64_t startTs; // Object created and added into the message queue
int64_t endTs; // the timestamp when the task is completed int64_t endTs; // the timestamp when the task is completed
@ -101,8 +99,6 @@ void schedulerFreeJob(int64_t* job, int32_t errCode);
void schedulerDestroy(void); void schedulerDestroy(void);
void schdExecCallback(SExecResult* pResult, void* param, int32_t code);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -15,29 +15,29 @@
#ifndef _TSTREAMUPDATE_H_ #ifndef _TSTREAMUPDATE_H_
#define _TSTREAMUPDATE_H_ #define _TSTREAMUPDATE_H_
#include "storageapi.h"
#include "taosdef.h" #include "taosdef.h"
#include "tarray.h" #include "tarray.h"
#include "tcommon.h" #include "tcommon.h"
#include "tmsg.h" #include "tmsg.h"
#include "storageapi.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid); bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo *pInfo); void updateInfoDestroy(SUpdateInfo* pInfo);
void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo);
void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo);
int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count);
void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count);
bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -70,7 +70,6 @@ typedef int64_t SyncIndex;
typedef int64_t SyncTerm; typedef int64_t SyncTerm;
typedef struct SSyncNode SSyncNode; typedef struct SSyncNode SSyncNode;
typedef struct SWal SWal;
typedef struct SSyncRaftEntry SSyncRaftEntry; typedef struct SSyncRaftEntry SSyncRaftEntry;
typedef enum { typedef enum {
@ -238,7 +237,7 @@ typedef struct SSyncInfo {
int32_t batchSize; int32_t batchSize;
SSyncCfg syncCfg; SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN]; char path[TSDB_FILENAME_LEN];
SWal* pWal; struct SWal* pWal;
SSyncFSM* pFsm; SSyncFSM* pFsm;
SMsgCb* msgcb; SMsgCb* msgcb;
int32_t pingMs; int32_t pingMs;

View File

@ -28,6 +28,11 @@ typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag;
int32_t taosSendHttpReport(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, int32_t taosSendHttpReport(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen,
EHttpCompFlag flag); EHttpCompFlag flag);
int64_t taosInitHttpChan();
int32_t taosSendHttpReportByChan(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen,
EHttpCompFlag flag, int64_t chanId);
void taosDestroyHttpChan(int64_t chanId);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -53,6 +53,8 @@ void taosPrintBackTrace();
void taosMemoryTrim(int32_t size); void taosMemoryTrim(int32_t size);
void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); void *taosMemoryMallocAlign(uint32_t alignment, int64_t size);
#define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n))
#define taosMemoryFreeClear(ptr) \ #define taosMemoryFreeClear(ptr) \
do { \ do { \
if (ptr) { \ if (ptr) { \

View File

@ -55,10 +55,12 @@ typedef enum { M2C = 0, C2M } ConvType;
#define tstrncpy(dst, src, size) \ #define tstrncpy(dst, src, size) \
do { \ do { \
strncpy((dst), (src), (size)); \ (void)strncpy((dst), (src), (size)); \
(dst)[(size)-1] = 0; \ (dst)[(size)-1] = 0; \
} while (0) } while (0)
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
char *tstrdup(const char *src); char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4); int32_t taosUcs4len(TdUcs4 *ucs4);
int64_t taosStr2int64(const char *str); int64_t taosStr2int64(const char *str);

View File

@ -52,7 +52,7 @@ typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void
* @param param * @param param
* @param comparFn * @param comparFn
*/ */
void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn); int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn);
/** /**
* Non-recursive quick sort. * Non-recursive quick sort.
@ -98,8 +98,8 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size
* @param maxroot: if heap is max root heap * @param maxroot: if heap is max root heap
* @return * @return
*/ */
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char *buf, bool maxroot); __ext_compar_fn_t compar, char *buf, bool maxroot);
/** /**
* sort heap to make sure it is a max/min root heap * sort heap to make sure it is a max/min root heap
@ -114,7 +114,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
* @param maxroot: if heap is max root heap * @param maxroot: if heap is max root heap
* @return * @return
*/ */
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, bool maxroot); int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
bool maxroot);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -89,6 +89,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x0022) // #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x0022) //
#define TSDB_CODE_RPC_NETWORK_ERROR TAOS_DEF_ERROR_CODE(0, 0x0023) #define TSDB_CODE_RPC_NETWORK_ERROR TAOS_DEF_ERROR_CODE(0, 0x0023)
#define TSDB_CODE_RPC_NETWORK_BUSY TAOS_DEF_ERROR_CODE(0, 0x0024) #define TSDB_CODE_RPC_NETWORK_BUSY TAOS_DEF_ERROR_CODE(0, 0x0024)
#define TSDB_CODE_HTTP_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0025)
@ -836,6 +837,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_PAR_TBNAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267E) #define TSDB_CODE_PAR_TBNAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267E)
#define TSDB_CODE_PAR_TAG_NAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267F) #define TSDB_CODE_PAR_TAG_NAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267F)
#define TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC TAOS_DEF_ERROR_CODE(0, 0x2680) #define TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC TAOS_DEF_ERROR_CODE(0, 0x2680)
#define TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2681)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner //planner
@ -940,6 +942,11 @@ int32_t taosGetErrSize();
// UTIL // UTIL
#define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000) #define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000)
// AUDIT
#define TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON TAOS_DEF_ERROR_CODE(0, 0x6100)
#define TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD TAOS_DEF_ERROR_CODE(0, 0x6101)
#define TSDB_CODE_AUDIT_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0x6102)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -205,7 +205,7 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
void taosArrayClearP(SArray* pArray, void (*fp)(void*)); void taosArrayClearP(SArray* pArray, void (*fp)(void*));
void* taosArrayDestroy(SArray* pArray); void taosArrayDestroy(SArray* pArray);
void taosArrayDestroyP(SArray* pArray, FDelete fp); void taosArrayDestroyP(SArray* pArray, FDelete fp);
@ -251,7 +251,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t
* @return * @return
*/ */
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); int32_t taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver); void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver);

View File

@ -19,6 +19,8 @@
#include "os.h" #include "os.h"
#include "tencode.h" #include "tencode.h"
#include "thash.h" #include "thash.h"
#include "tlog.h"
#include "tutil.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -35,19 +37,19 @@ typedef struct SBloomFilter {
uint64_t size; uint64_t size;
_hash_fn_t hashFn1; _hash_fn_t hashFn1;
_hash_fn_t hashFn2; _hash_fn_t hashFn2;
void *buffer; void* buffer;
double errorRate; double errorRate;
} SBloomFilter; } SBloomFilter;
SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate); int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF);
int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2);
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t h1, uint64_t h2);
void tBloomFilterDestroy(SBloomFilter *pBF); void tBloomFilterDestroy(SBloomFilter* pBF);
void tBloomFilterDump(const SBloomFilter *pBF); void tBloomFilterDump(const SBloomFilter* pBF);
bool tBloomFilterIsFull(const SBloomFilter *pBF); bool tBloomFilterIsFull(const SBloomFilter* pBF);
int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder);
SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder); int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -45,7 +45,10 @@ typedef struct SPatternCompareInfo {
TdUcs4 umatchOne; // unicode version matchOne TdUcs4 umatchOne; // unicode version matchOne
} SPatternCompareInfo; } SPatternCompareInfo;
int32_t InitRegexCache();
void DestroyRegexCache();
int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo); int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo);
int32_t checkRegexPattern(const char *pPattern);
int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo); int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo);
@ -83,7 +86,6 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight);
int32_t comparestrRegexMatch(const void *pLeft, const void *pRight); int32_t comparestrRegexMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight); int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight);
void DestoryThreadLocalRegComp();
int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight); int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight);
int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight); int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight);

View File

@ -18,6 +18,7 @@
#include "tcoding.h" #include "tcoding.h"
#include "tlist.h" #include "tlist.h"
#include "tutil.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -46,38 +47,8 @@ typedef struct {
SDecoderNode* dStack; SDecoderNode* dStack;
} SDecoder; } SDecoder;
#define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0] #define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos)
#define tRPut16(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[0];
#define tRPut32(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[0];
#define tRPut64(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[7]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[6]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[5]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[4]; \
((uint8_t*)(PDEST))[4] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[5] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[6] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[7] = ((uint8_t*)(PSRC))[0];
#define tRGet16 tRPut16
#define tRGet32 tRPut32
#define tRGet64 tRPut64
#define TD_CODER_POS(CODER) ((CODER)->pos)
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos)
#define tEncodeSize(E, S, SIZE, RET) \ #define tEncodeSize(E, S, SIZE, RET) \
do { \ do { \
@ -100,6 +71,7 @@ void tEncoderInit(SEncoder* pCoder, uint8_t* data, uint32_t size);
void tEncoderClear(SEncoder* pCoder); void tEncoderClear(SEncoder* pCoder);
int32_t tStartEncode(SEncoder* pCoder); int32_t tStartEncode(SEncoder* pCoder);
void tEndEncode(SEncoder* pCoder); void tEndEncode(SEncoder* pCoder);
static int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size);
static int32_t tEncodeU8(SEncoder* pCoder, uint8_t val); static int32_t tEncodeU8(SEncoder* pCoder, uint8_t val);
static int32_t tEncodeI8(SEncoder* pCoder, int8_t val); static int32_t tEncodeI8(SEncoder* pCoder, int8_t val);
static int32_t tEncodeU16(SEncoder* pCoder, uint16_t val); static int32_t tEncodeU16(SEncoder* pCoder, uint16_t val);
@ -127,6 +99,7 @@ void tDecoderClear(SDecoder* SDecoder);
int32_t tStartDecode(SDecoder* pCoder); int32_t tStartDecode(SDecoder* pCoder);
void tEndDecode(SDecoder* pCoder); void tEndDecode(SDecoder* pCoder);
static bool tDecodeIsEnd(SDecoder* pCoder); static bool tDecodeIsEnd(SDecoder* pCoder);
static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size);
static int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val); static int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val);
static int32_t tDecodeI8(SDecoder* pCoder, int8_t* val); static int32_t tDecodeI8(SDecoder* pCoder, int8_t* val);
static int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val); static int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val);
@ -149,97 +122,65 @@ static int32_t tDecodeCStr(SDecoder* pCoder, char** val);
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val);
/* ------------------------ IMPL ------------------------ */ /* ------------------------ IMPL ------------------------ */
#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ static FORCE_INLINE int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size) {
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) return -1; \
tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL)); \
} \
TD_CODER_MOVE_POS(CODER, sizeof(VAL)); \
return 0;
#define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \
while ((VAL) >= ENCODE_LIMIT) { \
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \
} \
\
(VAL) >>= 7; \
TD_CODER_MOVE_POS(CODER, 1); \
} \
\
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TD_CODER_CURRENT(CODER)[0] = (uint8_t)(VAL); \
} \
TD_CODER_MOVE_POS(CODER, 1); \
return 0;
#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS) \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) return -1; \
tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL)); \
TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \
return 0;
#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \
int32_t i = 0; \
*(PVAL) = 0; \
for (;;) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TYPE tval = TD_CODER_CURRENT(CODER)[0]; \
if (tval < ENCODE_LIMIT) { \
*(PVAL) |= (tval << (7 * i)); \
TD_CODER_MOVE_POS(pCoder, 1); \
break; \
} else { \
*(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \
i++; \
TD_CODER_MOVE_POS(pCoder, 1); \
} \
} \
\
return 0;
// 8
static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) {
if (pCoder->data) { if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1; if (pCoder->pos + size > pCoder->size) {
tPut(uint8_t, TD_CODER_CURRENT(pCoder), val); TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
memcpy(pCoder->data + pCoder->pos, val, size);
} }
TD_CODER_MOVE_POS(pCoder, sizeof(val));
pCoder->pos += size;
return 0; return 0;
} }
static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); }
if (pCoder->data) { static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); }
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1; static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) {
tPut(int8_t, TD_CODER_CURRENT(pCoder), val); return tEncodeFixed(pCoder, &val, sizeof(val));
} }
TD_CODER_MOVE_POS(pCoder, sizeof(val)); static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) {
return 0; return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) {
while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
} }
// 16
static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) { TD_ENCODE_MACRO(pCoder, val, uint16_t, 16); }
static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) { TD_ENCODE_MACRO(pCoder, val, int16_t, 16); }
// 32
static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) { TD_ENCODE_MACRO(pCoder, val, uint32_t, 32); }
static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) { TD_ENCODE_MACRO(pCoder, val, int32_t, 32); }
// 64
static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) { TD_ENCODE_MACRO(pCoder, val, uint64_t, 64); }
static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) { TD_ENCODE_MACRO(pCoder, val, int64_t, 64); }
// 16v
static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) { static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) {
return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val)); return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val));
} }
// 32v static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) {
static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) { static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) {
return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val)); return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val));
} }
// 64v static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) {
static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) { static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) {
return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val)); return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val));
} }
@ -265,14 +206,16 @@ static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) {
} }
static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) { static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) {
if (tEncodeU32v(pCoder, len) < 0) return -1; TAOS_CHECK_RETURN(tEncodeU32v(pCoder, len));
if (len) { if (len) {
if (pCoder->data) { if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) return -1; if (pCoder->pos + len > pCoder->size) {
memcpy(TD_CODER_CURRENT(pCoder), val, len); TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
memcpy(pCoder->data + pCoder->pos, val, len);
} }
TD_CODER_MOVE_POS(pCoder, len); pCoder->pos += len;
} }
return 0; return 0;
} }
@ -286,70 +229,137 @@ static FORCE_INLINE int32_t tEncodeCStr(SEncoder* pCoder, const char* val) {
} }
/* ------------------------ FOR DECODER ------------------------ */ /* ------------------------ FOR DECODER ------------------------ */
// 8 static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size) {
static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) { if (pCoder->pos + size > pCoder->size) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
tGet(uint8_t, TD_CODER_CURRENT(pCoder), *val); } else if (val) {
TD_CODER_MOVE_POS(pCoder, sizeof(*val)); memcpy(val, pCoder->data + pCoder->pos, size);
}
pCoder->pos += size;
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1; return tDecodeFixed(pCoder, val, sizeof(*val));
tGet(int8_t, TD_CODER_CURRENT(pCoder), *val);
TD_CODER_MOVE_POS(pCoder, sizeof(*val));
return 0;
} }
static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { return tDecodeFixed(pCoder, val, sizeof(*val)); }
// 16 // 16
static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) { TD_DECODE_MACRO(pCoder, val, uint16_t, 16); } static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) {
static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) { TD_DECODE_MACRO(pCoder, val, int16_t, 16); } return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 32 // 32
static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) { TD_DECODE_MACRO(pCoder, val, uint32_t, 32); } static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) {
static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) { TD_DECODE_MACRO(pCoder, val, int32_t, 32); } return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 64 // 64
static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) { TD_DECODE_MACRO(pCoder, val, uint64_t, 64); } static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) {
static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) { TD_DECODE_MACRO(pCoder, val, int64_t, 64); } return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 16v // 16v
static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) { static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint16_t); uint8_t byte;
uint16_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint16_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint16_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
} }
static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) { static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) {
uint16_t tval; uint16_t tval;
if (tDecodeU16v(pCoder, &tval) < 0) { TAOS_CHECK_RETURN(tDecodeU16v(pCoder, &tval));
return -1;
if (val) {
*val = ZIGZAGD(int16_t, tval);
} }
if (val) *val = ZIGZAGD(int16_t, tval);
return 0; return 0;
} }
// 32v // 32v
static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) { static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint32_t); uint8_t byte;
uint32_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint32_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint32_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
} }
static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) { static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) {
uint32_t tval; uint32_t tval;
if (tDecodeU32v(pCoder, &tval) < 0) { TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &tval));
return -1;
if (val) {
*val = ZIGZAGD(int32_t, tval);
} }
if (val) *val = ZIGZAGD(int32_t, tval);
return 0; return 0;
} }
// 64v // 64v
static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) { static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint64_t); uint8_t byte;
uint64_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint64_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint64_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
} }
static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) { static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) {
uint64_t tval; uint64_t tval;
if (tDecodeU64v(pCoder, &tval) < 0) { TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &tval));
return -1;
if (val) {
*val = ZIGZAGD(int64_t, tval);
} }
if (val) *val = ZIGZAGD(int64_t, tval);
return 0; return 0;
} }
@ -359,11 +369,11 @@ static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) {
float f; float f;
} v; } v;
if (tDecodeU32(pCoder, &(v.ui)) < 0) { TAOS_CHECK_RETURN(tDecodeU32(pCoder, &(v.ui)));
return -1;
}
*val = v.f; if (val) {
*val = v.f;
}
return 0; return 0;
} }
@ -373,30 +383,36 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) {
double d; double d;
} v; } v;
if (tDecodeU64(pCoder, &(v.ui)) < 0) { TAOS_CHECK_RETURN(tDecodeU64(pCoder, &(v.ui)));
return -1;
}
*val = v.d; if (val) {
*val = v.d;
}
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) {
uint32_t length = 0; uint32_t length = 0;
if (tDecodeU32v(pCoder, &length) < 0) return -1;
if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length));
if (val) { if (len) {
*val = (uint8_t*)TD_CODER_CURRENT(pCoder); *len = length;
} }
TD_CODER_MOVE_POS(pCoder, length); if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
if (val) {
*val = pCoder->data + pCoder->pos;
}
pCoder->pos += length;
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) { static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) {
if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1; TAOS_CHECK_RETURN(tDecodeBinary(pCoder, (uint8_t**)val, len));
(*len) -= 1; (*len) -= 1;
return 0; return 0;
} }
@ -409,7 +425,7 @@ static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) {
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
char* pStr; char* pStr;
uint32_t len; uint32_t len;
if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len));
memcpy(val, pStr, len + 1); memcpy(val, pStr, len + 1);
return 0; return 0;
@ -417,16 +433,22 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) { static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) {
uint64_t length = 0; uint64_t length = 0;
if (tDecodeU64v(pCoder, &length) < 0) return -1; TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &length));
if (length) { if (length) {
if (len) *len = length; if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; if (pCoder->pos + length > pCoder->size) {
*val = taosMemoryMalloc(length); TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
if (*val == NULL) return -1; }
memcpy(*val, TD_CODER_CURRENT(pCoder), length);
TD_CODER_MOVE_POS(pCoder, length); *val = taosMemoryMalloc(length);
if (*val == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*val, pCoder->data + pCoder->pos, length);
pCoder->pos += length;
} else { } else {
*val = NULL; *val = NULL;
} }
@ -435,16 +457,20 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin
static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) { static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) {
uint32_t length = 0; uint32_t length = 0;
if (tDecodeU32v(pCoder, &length) < 0) return -1; TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length));
if (length) { if (length) {
if (len) *len = length; if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
*val = taosMemoryMalloc(length); *val = taosMemoryMalloc(length);
if (*val == NULL) return -1; if (*val == NULL) {
memcpy(*val, TD_CODER_CURRENT(pCoder), length); TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*val, pCoder->data + pCoder->pos, length);
TD_CODER_MOVE_POS(pCoder, length); pCoder->pos += length;
} else { } else {
*val = NULL; *val = NULL;
} }
@ -452,7 +478,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u
} }
static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) { static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) {
if (tDecodeBinaryAlloc(pCoder, (void**)val, len) < 0) return -1; TAOS_CHECK_RETURN(tDecodeBinaryAlloc(pCoder, (void**)val, len));
(*len) -= 1; (*len) -= 1;
return 0; return 0;
} }

View File

@ -23,22 +23,22 @@ extern "C" {
#endif #endif
typedef struct SScalableBf { typedef struct SScalableBf {
SArray *bfArray; // array of bloom filters SArray* bfArray; // array of bloom filters
uint32_t growth; uint32_t growth;
uint64_t numBits; uint64_t numBits;
uint32_t maxBloomFilters; uint32_t maxBloomFilters;
int8_t status; int8_t status;
_hash_fn_t hashFn1; _hash_fn_t hashFn1;
_hash_fn_t hashFn2; _hash_fn_t hashFn2;
} SScalableBf; } SScalableBf;
SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate); int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf);
int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf *pSBf); void tScalableBfDestroy(SScalableBf* pSBf);
int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder);
SScalableBf *tScalableBfDecode(SDecoder *pDecoder); int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -21,6 +21,7 @@
#include "tdef.h" #include "tdef.h"
#include "thash.h" #include "thash.h"
#include "tmd5.h" #include "tmd5.h"
#include "tutil.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -56,7 +57,7 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str);
void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen); void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen);
int32_t parseCfgReal(const char* str, double* out); int32_t parseCfgReal(const char *str, double *out);
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
T_MD5_CTX context; T_MD5_CTX context;
@ -84,9 +85,9 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
static FORCE_INLINE int32_t taosCreateMD5Hash(char *pBuf, int32_t len) { static FORCE_INLINE int32_t taosCreateMD5Hash(char *pBuf, int32_t len) {
T_MD5_CTX ctx; T_MD5_CTX ctx;
tMD5Init(&ctx); tMD5Init(&ctx);
tMD5Update(&ctx, (uint8_t*)pBuf, len); tMD5Update(&ctx, (uint8_t *)pBuf, len);
tMD5Final(&ctx); tMD5Final(&ctx);
char* p = pBuf; char *p = pBuf;
int32_t resLen = 0; int32_t resLen = 0;
for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) {
resLen += snprintf(p, 3, "%02x", ctx.digest[i]); resLen += snprintf(p, 3, "%02x", ctx.digest[i]);
@ -147,6 +148,30 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member))) #define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member)))
#define TAOS_RETURN(code) \
do { \
return (terrno = (code)); \
} while (0)
#define TAOS_CHECK_RETURN(CMD) \
do { \
int32_t code = (CMD); \
if (code != TSDB_CODE_SUCCESS) { \
TAOS_RETURN(code); \
} \
} while (0)
#define TAOS_CHECK_GOTO(CMD, LINO, LABEL) \
do { \
code = (CMD); \
if (code != TSDB_CODE_SUCCESS) { \
if (LINO) { \
*((int32_t *)(LINO)) = __LINE__; \
} \
goto LABEL; \
} \
} while (0)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -82,7 +82,7 @@ extern "C" {
#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ #define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH)) (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH))
#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) #define MOVE_FORWARD_ONE(sql, len) ((void)memmove((void *)((sql)-1), (sql), len))
#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ #define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \ for (int i = 1; i < keyLen; ++i) { \
@ -211,19 +211,19 @@ extern int64_t smlFactorS[];
typedef int32_t (*_equal_fn_sml)(const void *, const void *); typedef int32_t (*_equal_fn_sml)(const void *, const void *);
SSmlHandle *smlBuildSmlInfo(TAOS *taos); int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle);
void smlDestroyInfo(SSmlHandle *info); void smlDestroyInfo(SSmlHandle *info);
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo);
SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta** sMeta);
int32_t smlSetCTableName(SSmlTableInfo *oneTable); int32_t smlSetCTableName(SSmlTableInfo *oneTable);
void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo);
STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); int32_t smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen, STableMeta **pTableMeta);
int32_t is_same_child_table_telnet(const void *a, const void *b); int32_t is_same_child_table_telnet(const void *a, const void *b);
int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len);
int32_t smlClearForRerun(SSmlHandle *info); int32_t smlClearForRerun(SSmlHandle *info);
@ -236,7 +236,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseJSON(SSmlHandle *info, char *payload); int32_t smlParseJSON(SSmlHandle *info, char *payload);
SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta);
bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv); bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);

View File

@ -127,7 +127,7 @@ static void generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_
cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code))); cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code)));
cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType)); cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType));
cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows)); cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows));
if(strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){ if(pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){
char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen]; char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen];
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0'; pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0';
cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)); cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr));

View File

@ -1398,7 +1398,8 @@ void hbMgrCleanUp() {
taosThreadMutexLock(&clientHbMgr.lock); taosThreadMutexLock(&clientHbMgr.lock);
appHbMgrCleanup(); appHbMgrCleanup();
clientHbMgr.appHbMgrs = taosArrayDestroy(clientHbMgr.appHbMgrs); taosArrayDestroy(clientHbMgr.appHbMgrs);
clientHbMgr.appHbMgrs = NULL;
taosThreadMutexUnlock(&clientHbMgr.lock); taosThreadMutexUnlock(&clientHbMgr.lock);
} }

View File

@ -1572,9 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
} }
// uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid); // uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid);
pQuery = smlInitHandle(); code = smlInitHandle(&pQuery);
if (pQuery == NULL) { if (code != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
@ -1656,9 +1655,9 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
pQuery = smlInitHandle();
if (pQuery == NULL) { code = smlInitHandle(&pQuery);
code = TSDB_CODE_OUT_OF_MEMORY; if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
@ -1755,12 +1754,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
conn.requestObjRefId = pRequest->self; conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle(); code = smlInitHandle(&pQuery);
if (pQuery == NULL) { if (code != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_OUT_OF_MEMORY;
SET_ERROR_MSG("init sml handle failed"); SET_ERROR_MSG("init sml handle failed");
goto end; goto end;
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter);
@ -1903,12 +1902,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
conn.requestObjRefId = pRequest->self; conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle(); code = smlInitHandle(&pQuery);
if (pQuery == NULL) { if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("init sml handle failed"); SET_ERROR_MSG("init sml handle failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum);

File diff suppressed because it is too large Load Diff

View File

@ -29,29 +29,29 @@
(start)++; \ (start)++; \
} }
static char *smlJsonGetObj(char *payload) { static int32_t smlJsonGetObj(char **payload) {
int leftBracketCnt = 0; int leftBracketCnt = 0;
bool isInQuote = false; bool isInQuote = false;
while (*payload) { while (**payload) {
if (*payload == '"' && *(payload - 1) != '\\') { if (**payload == '"' && *((*payload) - 1) != '\\') {
isInQuote = !isInQuote; isInQuote = !isInQuote;
} else if (!isInQuote && unlikely(*payload == '{')) { } else if (!isInQuote && unlikely(**payload == '{')) {
leftBracketCnt++; leftBracketCnt++;
payload++; (*payload)++;
continue; continue;
} else if (!isInQuote && unlikely(*payload == '}')) { } else if (!isInQuote && unlikely(**payload == '}')) {
leftBracketCnt--; leftBracketCnt--;
payload++; (*payload)++;
if (leftBracketCnt == 0) { if (leftBracketCnt == 0) {
return payload; return 0;
} else if (leftBracketCnt < 0) { } else if (leftBracketCnt < 0) {
return NULL; return -1;
} }
continue; continue;
} }
payload++; (*payload)++;
} }
return NULL; return -1;
} }
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
@ -99,8 +99,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
offset[index++] = *start - sTmp; offset[index++] = *start - sTmp;
element->timestamp = (*start); element->timestamp = (*start);
if (*(*start) == '{') { if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start); element->timestampLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -127,8 +128,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
offset[index++] = *start - sTmp; offset[index++] = *start - sTmp;
element->cols = (*start); element->cols = (*start);
if (*(*start) == '{') { if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start); element->colsLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -153,8 +155,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
JUMP_JSON_SPACE((*start)) JUMP_JSON_SPACE((*start))
offset[index++] = *start - sTmp; offset[index++] = *start - sTmp;
element->tags = (*start); element->tags = (*start);
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start); element->tagsLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -209,8 +212,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
(*start) += offset[index++]; (*start) += offset[index++];
element->timestamp = *start; element->timestamp = *start;
if (*(*start) == '{') { if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start); element->timestampLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -227,8 +231,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
(*start) += offset[index++]; (*start) += offset[index++];
element->cols = *start; element->cols = *start;
if (*(*start) == '{') { if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start); element->colsLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -244,8 +249,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
} else if ((*start)[1] == 't' && (*start)[2] == 'a') { } else if ((*start)[1] == 't' && (*start)[2] == 'a') {
(*start) += offset[index++]; (*start) += offset[index++];
element->tags = (*start); element->tags = (*start);
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start); element->tagsLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -261,7 +267,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
uError("elements != %d", OTD_JSON_FIELDS_NUM); uError("elements != %d", OTD_JSON_FIELDS_NUM);
return TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) {
@ -466,7 +472,11 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
break; break;
} }
case cJSON_String: { case cJSON_String: {
smlConvertJSONString(kv, "binary", root); int32_t ret = smlConvertJSONString(kv, "binary", root);
if (ret != TSDB_CODE_SUCCESS) {
uError("OTD:Failed to parse binary value from JSON Obj");
return ret;
}
break; break;
} }
case cJSON_Object: { case cJSON_Object: {
@ -492,20 +502,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
int32_t tagNum = cJSON_GetArraySize(tags); int32_t tagNum = cJSON_GetArraySize(tags);
if (unlikely(tagNum == 0)) { if (unlikely(tagNum == 0)) {
uError("SML:Tag should not be empty"); uError("SML:Tag should not be empty");
terrno = TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
return -1;
} }
for (int32_t i = 0; i < tagNum; ++i) { for (int32_t i = 0; i < tagNum; ++i) {
cJSON *tag = cJSON_GetArrayItem(tags, i); cJSON *tag = cJSON_GetArrayItem(tags, i);
if (unlikely(tag == NULL)) { if (unlikely(tag == NULL)) {
terrno = TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
return -1;
} }
size_t keyLen = strlen(tag->string); size_t keyLen = strlen(tag->string);
if (unlikely(IS_INVALID_COL_LEN(keyLen))) { if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
uError("OTD:Tag key length is 0 or too large than 64"); uError("OTD:Tag key length is 0 or too large than 64");
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
} }
// add kv to SSmlKv // add kv to SSmlKv
@ -516,19 +523,19 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
// value // value
int32_t ret = smlParseValueFromJSON(tag, &kv); int32_t ret = smlParseValueFromJSON(tag, &kv);
if (unlikely(ret != TSDB_CODE_SUCCESS)) { if (unlikely(ret != TSDB_CODE_SUCCESS)) {
terrno = ret; return ret;
return -1; }
if (taosArrayPush(preLineKV, &kv) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(preLineKV, &kv);
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS; return TSDB_CODE_TSC_INVALID_JSON;
return -1;
} }
cnt++; cnt++;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
@ -536,12 +543,18 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
if(info->dataFormat){ if(info->dataFormat){
ret = smlProcessSuperTable(info, elements); ret = smlProcessSuperTable(info, elements);
if(ret != 0){ if(ret != 0){
return terrno; if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
} }
ret = smlProcessTagJson(info, tags); ret = smlProcessTagJson(info, tags);
if(ret != 0){ if(ret != 0){
return terrno; if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
ret = smlJoinMeasureTag(elements); ret = smlJoinMeasureTag(elements);
if(ret != 0){ if(ret != 0){
@ -703,6 +716,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
// Parse tags // Parse tags
bool needFree = info->dataFormat; bool needFree = info->dataFormat;
elements->tags = cJSON_PrintUnformatted(tagsJson); elements->tags = cJSON_PrintUnformatted(tagsJson);
if (elements->tags == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
elements->tagsLen = strlen(elements->tags); elements->tagsLen = strlen(elements->tags);
if (is_same_child_table_telnet(elements, &info->preLine) != 0) { if (is_same_child_table_telnet(elements, &info->preLine) != 0) {
ret = smlParseTagsFromJSON(info, tagsJson, elements); ret = smlParseTagsFromJSON(info, tagsJson, elements);
@ -729,7 +745,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
// notice!!! put ts back to tag to ensure get meta->precision // notice!!! put ts back to tag to ensure get meta->precision
int64_t ts = smlParseTSFromJSON(info, tsJson); int64_t ts = smlParseTSFromJSON(info, tsJson);
if (unlikely(ts < 0)) { if (unlikely(ts < 0)) {
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); char* tmp = cJSON_PrintUnformatted(tsJson);
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts);
return TSDB_CODE_INVALID_TIMESTAMP; return TSDB_CODE_INVALID_TIMESTAMP;
} }
SSmlKv kvTs = {0}; SSmlKv kvTs = {0};
@ -823,7 +840,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
} }
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_VALUE; return ret;
} }
if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS;
@ -844,18 +861,23 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
cJSON *valueJson = cJSON_Parse(elements->cols); cJSON *valueJson = cJSON_Parse(elements->cols);
if (unlikely(valueJson == NULL)) { if (unlikely(valueJson == NULL)) {
uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
} }
taosArrayPush(info->tagJsonArray, &valueJson); if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){
cJSON_Delete(valueJson);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_OUT_OF_MEMORY;
}
ret = smlParseValueFromJSONObj(valueJson, &kv); ret = smlParseValueFromJSONObj(valueJson, &kv);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
uError("SML:Failed to parse value from JSON Obj:%s", elements->cols); uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp; elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
elements->cols[elements->colsLen] = tmp; elements->cols[elements->colsLen] = tmp;
} else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) {
uError("SML:cols invalidate:%s", elements->cols); uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols);
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
@ -870,7 +892,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
return TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
} }
taosArrayPush(info->tagJsonArray, &tagsJson); if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){
cJSON_Delete(tagsJson);
uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id);
return TSDB_CODE_OUT_OF_MEMORY;
}
ret = smlParseTagsFromJSON(info, tagsJson, elements); ret = smlParseTagsFromJSON(info, tagsJson, elements);
if (unlikely(ret)) { if (unlikely(ret)) {
uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id);
@ -937,7 +963,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) {
return ret; return ret;
} }
info->lines = (SSmlLineInfo *)tmp; info->lines = (SSmlLineInfo *)tmp;
memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo));
} }
ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt);
if ((info->lines + cnt)->measure == NULL) break; if ((info->lines + cnt)->measure == NULL) break;

View File

@ -103,7 +103,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
return code; return code;
} }
char* tmp = taosMemoryCalloc(pVal->length, 1); char* tmp = taosMemoryCalloc(pVal->length, 1);
memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN);
code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length); code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length);
taosMemoryFree(tmp); taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -149,7 +152,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
if(data == NULL){ if(data == NULL){
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); (void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length);
pVal->value = data; pVal->value = data;
} }
@ -212,8 +215,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
while (*sql < sqlEnd) { while (*sql < sqlEnd) {
if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
if (unlikely(IS_EQUAL(*sql,escapeChar))) { if (unlikely(IS_EQUAL(*sql,escapeChar))) {
keyLen = *sql - key; keyLen = *sql - key;
@ -230,8 +232,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
} }
// parse value // parse value
@ -245,8 +246,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
break; break;
} else if (unlikely(IS_EQUAL(*sql,escapeChar))) { } else if (unlikely(IS_EQUAL(*sql,escapeChar))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
@ -261,24 +261,28 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
if (unlikely(valueLen == 0)) { if (unlikely(valueLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
return -1;
} }
if (keyEscaped) { if (keyEscaped) {
char *tmp = (char *)taosMemoryMalloc(keyLen); char *tmp = (char *)taosMemoryMalloc(keyLen);
memcpy(tmp, key, keyLen); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, key, keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen);
key = tmp; key = tmp;
} }
if (valueEscaped) { if (valueEscaped) {
char *tmp = (char *)taosMemoryMalloc(valueLen); char *tmp = (char *)taosMemoryMalloc(valueLen);
memcpy(tmp, value, valueLen); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, value, valueLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen);
value = tmp; value = tmp;
} }
@ -289,11 +293,12 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
.length = valueLen, .length = valueLen,
.keyEscaped = keyEscaped, .keyEscaped = keyEscaped,
.valueEscaped = valueEscaped}; .valueEscaped = valueEscaped};
taosArrayPush(preLineKV, &kv); if(taosArrayPush(preLineKV, &kv) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS; return TSDB_CODE_TSC_INVALID_JSON;
return -1;
} }
cnt++; cnt++;
@ -302,7 +307,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
} }
(*sql)++; (*sql)++;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
@ -315,13 +320,19 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if(info->dataFormat){ if(info->dataFormat){
ret = smlProcessSuperTable(info, elements); ret = smlProcessSuperTable(info, elements);
if(ret != 0){ if(ret != 0){
return terrno; if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
} }
ret = smlProcessTagLine(info, sql, sqlEnd); ret = smlProcessTagLine(info, sql, sqlEnd);
if(ret != 0){ if(ret != 0){
return terrno; if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
return smlProcessChildTable(info, elements); return smlProcessChildTable(info, elements);
@ -410,7 +421,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if (keyEscaped) { if (keyEscaped) {
char *tmp = (char *)taosMemoryMalloc(kv.keyLen); char *tmp = (char *)taosMemoryMalloc(kv.keyLen);
memcpy(tmp, key, kv.keyLen); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, key, kv.keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen);
kv.key = tmp; kv.key = tmp;
kv.keyEscaped = keyEscaped; kv.keyEscaped = keyEscaped;
@ -418,7 +432,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if (valueEscaped) { if (valueEscaped) {
char *tmp = (char *)taosMemoryMalloc(kv.length); char *tmp = (char *)taosMemoryMalloc(kv.length);
memcpy(tmp, kv.value, kv.length); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, kv.value, kv.length);
PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY); ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY);
if(kv.type == TSDB_DATA_TYPE_VARBINARY){ if(kv.type == TSDB_DATA_TYPE_VARBINARY){
@ -437,8 +454,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
} else { } else {
if (currElement->colArray == NULL) { if (currElement->colArray == NULL) {
currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
if (currElement->colArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp
return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(currElement->colArray, &kv); // reserve for timestamp
} }
cnt++; cnt++;

View File

@ -84,8 +84,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
while (sql < sqlEnd) { while (sql < sqlEnd) {
if (unlikely(*sql == SPACE)) { if (unlikely(*sql == SPACE)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
if (unlikely(*sql == EQUAL)) { if (unlikely(*sql == EQUAL)) {
keyLen = sql - key; keyLen = sql - key;
@ -97,8 +96,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
if (unlikely(IS_INVALID_COL_LEN(keyLen))) { if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
} }
// parse value // parse value
@ -111,8 +109,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
} }
if (unlikely(*sql == EQUAL)) { if (unlikely(*sql == EQUAL)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
sql++; sql++;
} }
@ -120,13 +117,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
if (unlikely(valueLen == 0)) { if (unlikely(valueLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
terrno = TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
return -1;
} }
if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
return -1;
} }
SSmlKv kv = {.key = key, SSmlKv kv = {.key = key,
@ -136,14 +131,15 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
.length = valueLen, .length = valueLen,
.keyEscaped = false, .keyEscaped = false,
.valueEscaped = false}; .valueEscaped = false};
taosArrayPush(preLineKV, &kv); if (taosArrayPush(preLineKV, &kv) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
cnt++; cnt++;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) { static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
@ -156,13 +152,19 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
if(info->dataFormat){ if(info->dataFormat){
ret = smlProcessSuperTable(info, elements); ret = smlProcessSuperTable(info, elements);
if(ret != 0){ if(ret != 0){
return terrno; if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
} }
ret = smlProcessTagTelnet(info, data, sqlEnd); ret = smlProcessTagTelnet(info, data, sqlEnd);
if(ret != 0){ if(ret != 0){
return terrno; if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
ret = smlJoinMeasureTag(elements); ret = smlJoinMeasureTag(elements);

View File

@ -40,13 +40,16 @@ TEST(testCase, smlParseInfluxString_Test) {
msgBuf.len = 256; msgBuf.len = 256;
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, 0);
info->protocol = TSDB_SML_LINE_PROTOCOL; info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false; info->dataFormat = false;
// case 1 // case 1
char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3";
char *sql = (char *)taosMemoryCalloc(256, 1); char *sql = (char *)taosMemoryCalloc(256, 1);
memcpy(sql, tmp, strlen(tmp) + 1); ASSERT_NE(sql, nullptr);
(void)memcpy(sql, tmp, strlen(tmp) + 1);
int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
@ -62,22 +65,22 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 2 false // case 2 false
tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 4 tag is null // case 4 tag is null
tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
@ -92,13 +95,13 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 5 tag is null // case 5 tag is null
tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql + 1); ASSERT_EQ(elements.measure, sql + 1);
@ -112,36 +115,36 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 6 // case 6
tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 ";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
smlClearForRerun(info); ASSERT_EQ(smlClearForRerun(info), 0);
// case 7 // case 7
tmp = " st , "; tmp = " st , ";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 8 false // case 8 false
tmp = ", st , "; tmp = ", st , ";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
taosMemoryFree(sql); taosMemoryFree(sql);
smlDestroyInfo(info); smlDestroyInfo(info);
@ -228,7 +231,9 @@ TEST(testCase, smlParseCols_Error_Test) {
"escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000" "escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000"
}; };
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->protocol = TSDB_SML_LINE_PROTOCOL; info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false; info->dataFormat = false;
for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) {
@ -238,13 +243,14 @@ TEST(testCase, smlParseCols_Error_Test) {
msgBuf.len = 256; msgBuf.len = 256;
int32_t len = strlen(data[i]); int32_t len = strlen(data[i]);
char *sql = (char *)taosMemoryCalloc(256, 1); char *sql = (char *)taosMemoryCalloc(256, 1);
memcpy(sql, data[i], len + 1); ASSERT_NE(sql, nullptr);
(void)memcpy(sql, data[i], len + 1);
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements);
// printf("i:%d\n", i); // printf("i:%d\n", i);
ASSERT_NE(ret, TSDB_CODE_SUCCESS); ASSERT_NE(ret, TSDB_CODE_SUCCESS);
taosMemoryFree(sql); taosMemoryFree(sql);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
} }
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -254,7 +260,9 @@ TEST(testCase, smlParseCols_Test) {
SSmlMsgBuf msgBuf; SSmlMsgBuf msgBuf;
msgBuf.buf = msg; msgBuf.buf = msg;
msgBuf.len = 256; msgBuf.len = 256;
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->protocol = TSDB_SML_LINE_PROTOCOL; info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false; info->dataFormat = false;
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
@ -267,7 +275,8 @@ TEST(testCase, smlParseCols_Test) {
"cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000";
int32_t len = strlen(data); int32_t len = strlen(data);
char *sql = (char *)taosMemoryCalloc(1024, 1); char *sql = (char *)taosMemoryCalloc(1024, 1);
memcpy(sql, data, len + 1); ASSERT_NE(sql, nullptr);
(void)memcpy(sql, data, len + 1);
int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements);
ASSERT_EQ(ret, TSDB_CODE_SUCCESS); ASSERT_EQ(ret, TSDB_CODE_SUCCESS);
int32_t size = taosArrayGetSize(elements.colArray); int32_t size = taosArrayGetSize(elements.colArray);
@ -275,6 +284,7 @@ TEST(testCase, smlParseCols_Test) {
// binary // binary
SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY);
@ -285,6 +295,7 @@ TEST(testCase, smlParseCols_Test) {
// nchar // nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); kv = (SSmlKv *)taosArrayGet(elements.colArray, 2);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
@ -293,6 +304,7 @@ TEST(testCase, smlParseCols_Test) {
// bool // bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); kv = (SSmlKv *)taosArrayGet(elements.colArray, 3);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -301,6 +313,7 @@ TEST(testCase, smlParseCols_Test) {
// double // double
kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); kv = (SSmlKv *)taosArrayGet(elements.colArray, 4);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
@ -310,6 +323,7 @@ TEST(testCase, smlParseCols_Test) {
// float // float
kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); kv = (SSmlKv *)taosArrayGet(elements.colArray, 5);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
@ -319,6 +333,7 @@ TEST(testCase, smlParseCols_Test) {
// float // float
kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); kv = (SSmlKv *)taosArrayGet(elements.colArray, 6);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT);
@ -328,6 +343,7 @@ TEST(testCase, smlParseCols_Test) {
// tiny int // tiny int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); kv = (SSmlKv *)taosArrayGet(elements.colArray, 7);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0);
ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->keyLen, 3);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT);
@ -336,6 +352,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned tiny int // unsigned tiny int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); kv = (SSmlKv *)taosArrayGet(elements.colArray, 8);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0);
ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->keyLen, 3);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT);
@ -344,6 +361,7 @@ TEST(testCase, smlParseCols_Test) {
// small int // small int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); kv = (SSmlKv *)taosArrayGet(elements.colArray, 9);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT);
@ -352,6 +370,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned smallint // unsigned smallint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); kv = (SSmlKv *)taosArrayGet(elements.colArray, 10);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT);
@ -360,6 +379,7 @@ TEST(testCase, smlParseCols_Test) {
// int // int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); kv = (SSmlKv *)taosArrayGet(elements.colArray, 11);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT);
@ -368,6 +388,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned int // unsigned int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); kv = (SSmlKv *)taosArrayGet(elements.colArray, 12);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT);
@ -376,6 +397,7 @@ TEST(testCase, smlParseCols_Test) {
// bigint // bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); kv = (SSmlKv *)taosArrayGet(elements.colArray, 13);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT);
@ -384,6 +406,7 @@ TEST(testCase, smlParseCols_Test) {
// bigint // bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); kv = (SSmlKv *)taosArrayGet(elements.colArray, 14);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0);
ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->keyLen, 2);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT);
@ -392,6 +415,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned bigint // unsigned bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); kv = (SSmlKv *)taosArrayGet(elements.colArray, 15);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT);
@ -400,6 +424,7 @@ TEST(testCase, smlParseCols_Test) {
// bool // bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); kv = (SSmlKv *)taosArrayGet(elements.colArray, 16);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0);
ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->keyLen, 9);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -408,6 +433,7 @@ TEST(testCase, smlParseCols_Test) {
// bool // bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); kv = (SSmlKv *)taosArrayGet(elements.colArray, 17);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0);
ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->keyLen, 6);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -416,6 +442,7 @@ TEST(testCase, smlParseCols_Test) {
// bool // bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); kv = (SSmlKv *)taosArrayGet(elements.colArray, 18);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0);
ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->keyLen, 6);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -424,13 +451,14 @@ TEST(testCase, smlParseCols_Test) {
// nchar // nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); kv = (SSmlKv *)taosArrayGet(elements.colArray, 19);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->length, 4);
ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
taosMemoryFree(sql); taosMemoryFree(sql);
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -471,7 +499,9 @@ TEST(testCase, smlParseNumber_Test) {
} }
TEST(testCase, smlParseTelnetLine_error_Test) { TEST(testCase, smlParseTelnetLine_error_Test) {
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->dataFormat = false; info->dataFormat = false;
info->protocol = TSDB_SML_TELNET_PROTOCOL; info->protocol = TSDB_SML_TELNET_PROTOCOL;
ASSERT_NE(info, nullptr); ASSERT_NE(info, nullptr);
@ -509,7 +539,9 @@ TEST(testCase, smlParseTelnetLine_error_Test) {
} }
TEST(testCase, smlParseTelnetLine_Test) { TEST(testCase, smlParseTelnetLine_Test) {
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->dataFormat = false; info->dataFormat = false;
info->protocol = TSDB_SML_TELNET_PROTOCOL; info->protocol = TSDB_SML_TELNET_PROTOCOL;
ASSERT_NE(info, nullptr); ASSERT_NE(info, nullptr);
@ -528,7 +560,9 @@ TEST(testCase, smlParseTelnetLine_Test) {
} }
//TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { //TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) {
// SSmlHandle *info = smlBuildSmlInfo(NULL); // SSmlHandle *info = nullptr;
// int32_t code = smlBuildSmlInfo(nullptr, &info);
// ASSERT_EQ(code, TSDB_CODE_SUCCESS);
// info->protocol = TSDB_SML_JSON_PROTOCOL; // info->protocol = TSDB_SML_JSON_PROTOCOL;
// ASSERT_NE(info, nullptr); // ASSERT_NE(info, nullptr);
// //
@ -568,7 +602,7 @@ TEST(testCase, smlParseNumber_performance_Test) {
for (int j = 0; j < 10000000; ++j) { for (int j = 0; j < 10000000; ++j) {
kv.value = str[i]; kv.value = str[i];
kv.length = strlen(str[i]); kv.length = strlen(str[i]);
smlParseNumber(&kv, &msgBuf); (void)smlParseNumber(&kv, &msgBuf); //ignore result
} }
printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1);
printf("\n"); printf("\n");
@ -576,7 +610,7 @@ TEST(testCase, smlParseNumber_performance_Test) {
for (int j = 0; j < 10000000; ++j) { for (int j = 0; j < 10000000; ++j) {
kv.value = str[i]; kv.value = str[i];
kv.length = strlen(str[i]); kv.length = strlen(str[i]);
smlParseNumberOld(&kv, &msgBuf); (void)smlParseNumberOld(&kv, &msgBuf); //ignore result
} }
printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2);
printf("\n\n"); printf("\n\n");

View File

@ -3,6 +3,7 @@
#include "cos.h" #include "cos.h"
#include "cos_cp.h" #include "cos_cp.h"
#include "tdef.h" #include "tdef.h"
#include "tutil.h"
extern char tsS3Endpoint[]; extern char tsS3Endpoint[];
extern char tsS3AccessKeyId[]; extern char tsS3AccessKeyId[];
@ -40,7 +41,7 @@ int32_t s3Begin() {
if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) {
uError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); uError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
protocolG = !tsS3Https; protocolG = !tsS3Https;
@ -48,12 +49,12 @@ int32_t s3Begin() {
uriStyleG = S3UriStyleVirtualHost; uriStyleG = S3UriStyleVirtualHost;
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
void s3End() { S3_deinitialize(); } void s3End() { S3_deinitialize(); }
int32_t s3Init() { return 0; /*s3Begin();*/ } int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); /*s3Begin();*/ }
void s3CleanUp() { /*s3End();*/ void s3CleanUp() { /*s3End();*/
} }
@ -61,7 +62,7 @@ void s3CleanUp() { /*s3End();*/
static int32_t s3ListBucket(char const *bucketname); static int32_t s3ListBucket(char const *bucketname);
int32_t s3CheckCfg() { int32_t s3CheckCfg() {
int32_t code = 0; int32_t code = 0, lino = 0;
if (!tsS3Enabled) { if (!tsS3Enabled) {
fprintf(stderr, "s3 not configured.\n"); fprintf(stderr, "s3 not configured.\n");
@ -71,7 +72,7 @@ int32_t s3CheckCfg() {
code = s3Begin(); code = s3Begin();
if (code != 0) { if (code != 0) {
fprintf(stderr, "failed to initialize s3.\n"); fprintf(stderr, "failed to initialize s3.\n");
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
// test put // test put
@ -91,20 +92,17 @@ int32_t s3CheckCfg() {
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (!fp) { if (!fp) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to open test file: %s.\n", path); fprintf(stderr, "failed to open test file: %s.\n", path);
// uError("ERROR: %s Failed to open %s", __func__, path); // uError("ERROR: %s Failed to open %s", __func__, path);
goto _exit; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
} }
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to write test file: %s.\n", path); fprintf(stderr, "failed to write test file: %s.\n", path);
goto _exit; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
} }
if (taosFsyncFile(fp) < 0) { if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to fsync test file: %s.\n", path); fprintf(stderr, "failed to fsync test file: %s.\n", path);
goto _exit; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
} }
taosCloseFile(&fp); taosCloseFile(&fp);
@ -112,7 +110,7 @@ int32_t s3CheckCfg() {
code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16); code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16);
if (code != 0) { if (code != 0) {
fprintf(stderr, "put object %s : failed.\n", objectname[0]); fprintf(stderr, "put object %s : failed.\n", objectname[0]);
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
fprintf(stderr, "put object %s: success.\n\n", objectname[0]); fprintf(stderr, "put object %s: success.\n\n", objectname[0]);
@ -121,7 +119,7 @@ int32_t s3CheckCfg() {
code = s3ListBucket(tsS3BucketName); code = s3ListBucket(tsS3BucketName);
if (code != 0) { if (code != 0) {
fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName); fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName);
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName); fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName);
@ -134,7 +132,7 @@ int32_t s3CheckCfg() {
code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock); code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
if (code != 0) { if (code != 0) {
fprintf(stderr, "get object %s : failed.\n", objectname[0]); fprintf(stderr, "get object %s : failed.\n", objectname[0]);
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
char buf[7] = {0}; char buf[7] = {0};
memcpy(buf, pBlock, c_len); memcpy(buf, pBlock, c_len);
@ -147,14 +145,14 @@ int32_t s3CheckCfg() {
code = s3DeleteObjects(objectname, 1); code = s3DeleteObjects(objectname, 1);
if (code != 0) { if (code != 0) {
fprintf(stderr, "delete object %s : failed.\n", objectname[0]); fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
fprintf(stderr, "delete object %s: success.\n\n", objectname[0]); fprintf(stderr, "delete object %s: success.\n\n", objectname[0]);
s3End(); s3End();
_exit: _exit:
return code; TAOS_RETURN(code);
} }
static int should_retry() { static int should_retry() {
@ -169,7 +167,7 @@ static int should_retry() {
} }
*/ */
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status, static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status,
@ -202,7 +200,6 @@ static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *prope
} }
static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) {
//(void)callbackData;
TS3SizeCBD *cbd = callbackData; TS3SizeCBD *cbd = callbackData;
if (properties->contentLength > 0) { if (properties->contentLength > 0) {
cbd->content_length = properties->contentLength; cbd->content_length = properties->contentLength;
@ -249,7 +246,7 @@ static int32_t s3ListBucket(char const *bucketname) {
SArray *objectArray = getListByPrefix("s3"); SArray *objectArray = getListByPrefix("s3");
if (objectArray == NULL) { if (objectArray == NULL) {
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
const char **object_name = TARRAY_DATA(objectArray); const char **object_name = TARRAY_DATA(objectArray);
@ -262,7 +259,7 @@ static int32_t s3ListBucket(char const *bucketname) {
taosArrayDestroyEx(objectArray, s3FreeObjectKey); taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return code; TAOS_RETURN(code);
} }
typedef struct growbuffer { typedef struct growbuffer {
@ -607,13 +604,13 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
} }
return code; TAOS_RETURN(code);
} }
static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name, static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name,
int64_t contentLength, S3PutProperties *put_prop, int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) { put_object_callback_data *data) {
int32_t code = 0; int32_t code = 0, lino = 0;
uint64_t totalContentLength = contentLength; uint64_t totalContentLength = contentLength;
uint64_t todoContentLength = contentLength; uint64_t todoContentLength = contentLength;
UploadManager manager = {0}; UploadManager manager = {0};
@ -640,6 +637,9 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *)); manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
if (!manager.etags) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
manager.next_etags_pos = 0; manager.next_etags_pos = 0;
do { do {
S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager); S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
@ -647,8 +647,7 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha
if (manager.upload_id == 0 || manager.status != S3StatusOK) { if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
upload: upload:
@ -672,8 +671,7 @@ upload:
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) { if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
contentLength -= chunk_size; contentLength -= chunk_size;
todoContentLength -= chunk_size; todoContentLength -= chunk_size;
@ -686,8 +684,7 @@ upload:
int n; int n;
for (i = 0; i < totalSeq; i++) { for (i = 0; i < totalSeq; i++) {
if (!manager.etags[i]) { if (!manager.etags[i]) {
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
n = snprintf(buf, sizeof(buf), n = snprintf(buf, sizeof(buf),
"<Part><PartNumber>%d</PartNumber>" "<Part><PartNumber>%d</PartNumber>"
@ -704,11 +701,13 @@ upload:
} while (S3_status_is_retryable(manager.status) && should_retry()); } while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) { if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
clean: _exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (manager.upload_id) { if (manager.upload_id) {
taosMemoryFree(manager.upload_id); taosMemoryFree(manager.upload_id);
} }
@ -718,13 +717,13 @@ clean:
growbuffer_destroy(manager.gb); growbuffer_destroy(manager.gb);
taosMemoryFree(manager.etags); taosMemoryFree(manager.etags);
return code; TAOS_RETURN(code);
} }
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime, static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
char const *object_name, int64_t contentLength, S3PutProperties *put_prop, char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) { put_object_callback_data *data) {
int32_t code = 0; int32_t code = 0, lino = 0;
uint64_t totalContentLength = contentLength; uint64_t totalContentLength = contentLength;
// uint64_t todoContentLength = contentLength; // uint64_t todoContentLength = contentLength;
@ -744,13 +743,16 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
SCheckpoint cp = {0}; SCheckpoint cp = {0};
cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart)); cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
if (!cp.parts) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (taosCheckExistFile(file_cp_path)) { if (taosCheckExistFile(file_cp_path)) {
if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) { if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) {
manager.upload_id = strdup(cp.upload_id); manager.upload_id = strdup(cp.upload_id);
need_init_upload = false; need_init_upload = false;
} else { } else {
cos_cp_remove(file_cp_path); TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit);
} }
} }
@ -763,16 +765,14 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
if (manager.upload_id == 0 || manager.status != S3StatusOK) { if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size); cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size);
} }
if (cos_cp_open(file_cp_path, &cp)) { if (cos_cp_open(file_cp_path, &cp)) {
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
int part_num = 0; int part_num = 0;
@ -791,6 +791,10 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *)); manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
if (!manager.etags) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
manager.next_etags_pos = 0; manager.next_etags_pos = 0;
upload: upload:
@ -802,8 +806,7 @@ upload:
if (i > 0 && cp.parts[i - 1].completed) { if (i > 0 && cp.parts[i - 1].completed) {
if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) { if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto clean;
} }
} }
@ -827,25 +830,23 @@ upload:
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) { if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
//(void)cos_cp_dump(&cp); //(void)cos_cp_dump(&cp);
goto clean;
} }
if (!manager.etags[seq - 1]) { if (!manager.etags[seq - 1]) {
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0); cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0);
(void)cos_cp_dump(&cp); TAOS_CHECK_GOTO(cos_cp_dump(&cp), &lino, _exit);
contentLength -= chunk_size; contentLength -= chunk_size;
// todoContentLength -= chunk_size; // todoContentLength -= chunk_size;
} }
cos_cp_close(cp.thefile); TAOS_CHECK_GOTO(cos_cp_close(cp.thefile), &lino, _exit);
cp.thefile = 0; cp.thefile = 0;
int size = 0; int size = 0;
@ -869,20 +870,23 @@ upload:
} while (S3_status_is_retryable(manager.status) && should_retry()); } while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) { if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
cos_cp_remove(file_cp_path); TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit);
clean: _exit:
/* /*
if (parts) { if (parts) {
taosMemoryFree(parts); taosMemoryFree(parts);
} }
*/ */
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (cp.thefile) { if (cp.thefile) {
cos_cp_close(cp.thefile); (void)cos_cp_close(cp.thefile);
} }
if (cp.parts) { if (cp.parts) {
taosMemoryFree(cp.parts); taosMemoryFree(cp.parts);
@ -899,7 +903,7 @@ clean:
taosMemoryFree(manager.etags); taosMemoryFree(manager.etags);
growbuffer_destroy(manager.gb); growbuffer_destroy(manager.gb);
return code; TAOS_RETURN(code);
} }
int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) { int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) {
@ -917,15 +921,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
put_object_callback_data data = {0}; put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) { if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file); uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file); uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
@ -956,7 +958,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
growbuffer_destroy(data.gb); growbuffer_destroy(data.gb);
} }
return code; TAOS_RETURN(code);
} }
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
@ -974,22 +976,19 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int
put_object_callback_data data = {0}; put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) { if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file); uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
contentLength = size; contentLength = size;
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file); uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) { if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) {
taosCloseFile(&data.infileFD); taosCloseFile(&data.infileFD);
code = TAOS_SYSTEM_ERROR(errno); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
return code;
} }
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
@ -1016,7 +1015,7 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int
growbuffer_destroy(data.gb); growbuffer_destroy(data.gb);
} }
return code; TAOS_RETURN(code);
} }
typedef struct list_bucket_callback_data { typedef struct list_bucket_callback_data {
@ -1130,11 +1129,11 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
code = -1; code = TSDB_CODE_FAILED;
} }
} }
return code; TAOS_RETURN(code);
} }
void s3DeleteObjectsByPrefix(const char *prefix) { void s3DeleteObjectsByPrefix(const char *prefix) {
@ -1187,17 +1186,19 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
if (cbd.status != S3StatusOK) { if (cbd.status != S3StatusOK) {
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
} }
if (check && cbd.buf_pos != size) { if (check && cbd.buf_pos != size) {
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
} }
*ppBlock = cbd.buf; *ppBlock = cbd.buf;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) { static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) {
@ -1218,9 +1219,8 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) {
TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); uError("[s3] open file error, errno:%d, fileName:%s", TAOS_SYSTEM_ERROR(errno), fileName);
uError("[s3] open file error, errno:%d, fileName:%s", terrno, fileName); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
return -1;
} }
TS3GetData cbd = {0}; TS3GetData cbd = {0};
@ -1232,16 +1232,17 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) {
if (cbd.status != S3StatusOK) { if (cbd.status != S3StatusOK) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
taosCloseFile(&pFile); taosCloseFile(&pFile);
return TAOS_SYSTEM_ERROR(EIO); TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
} }
taosCloseFile(&pFile); taosCloseFile(&pFile);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
SArray *objectArray = getListByPrefix(prefix); SArray *objectArray = getListByPrefix(prefix);
if (objectArray == NULL) return -1; if (objectArray == NULL) TAOS_RETURN(TSDB_CODE_FAILED);
for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) { for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) {
char *object = taosArrayGetP(objectArray, i); char *object = taosArrayGetP(objectArray, i);
@ -1255,7 +1256,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
} }
if (s3GetObjectToFile(object, fileName) != 0) { if (s3GetObjectToFile(object, fileName) != 0) {
taosArrayDestroyEx(objectArray, s3FreeObjectKey); taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
} }
taosArrayDestroyEx(objectArray, s3FreeObjectKey); taosArrayDestroyEx(objectArray, s3FreeObjectKey);
@ -1279,7 +1280,7 @@ long s3Size(const char *object_name) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
size = cbd.content_length; size = cbd.content_length;
@ -1306,7 +1307,7 @@ int32_t s3Init() {
// set log output, default stderr // set log output, default stderr
cos_log_set_output(NULL); cos_log_set_output(NULL);
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
void s3CleanUp() { cos_http_io_deinitialize(); } void s3CleanUp() { cos_http_io_deinitialize(); }
@ -1363,10 +1364,10 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
cos_pool_destroy(p); cos_pool_destroy(p);
if (s->code != 200) { if (s->code != 200) {
return code = s->code; TAOS_RETURN(s->code);
} }
return code; TAOS_RETURN(code);
} }
int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) { int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) {
@ -1405,10 +1406,10 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_
cos_pool_destroy(p); cos_pool_destroy(p);
if (s->code != 200) { if (s->code != 200) {
return code = s->code; TAOS_RETURN(s->code);
} }
return code; TAOS_RETURN(code);
} }
void s3DeleteObjectsByPrefix(const char *prefix_str) { void s3DeleteObjectsByPrefix(const char *prefix_str) {
@ -1465,7 +1466,7 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) {
cos_warn_log("delete objects failed\n"); cos_warn_log("delete objects failed\n");
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
bool s3Exists(const char *object_name) { bool s3Exists(const char *object_name) {
@ -1547,7 +1548,7 @@ bool s3Get(const char *object_name, const char *path) {
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) { int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) {
(void)check; (void)check;
int32_t code = 0; int32_t code = 0, lino = 0;
cos_pool_t *p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_status_t *s = NULL; cos_status_t *s = NULL;
@ -1588,10 +1589,9 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
s = cos_get_object_to_buffer(options, &bucket, &object, headers, NULL, &download_buffer, &resp_headers); s = cos_get_object_to_buffer(options, &bucket, &object, headers, NULL, &download_buffer, &resp_headers);
log_status(s); log_status(s);
if (!cos_status_is_ok(s)) { if (!cos_status_is_ok(s)) {
vError("s3: %d(%s)", s->code, s->error_msg); uError("s3: %d(%s)", s->code, s->error_msg);
vError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
code = TAOS_SYSTEM_ERROR(EIO); TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
return code;
} }
// print_headers(resp_headers); // print_headers(resp_headers);
@ -1601,6 +1601,10 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { len += cos_buf_size(content); } cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { len += cos_buf_size(content); }
// char *buf = cos_pcalloc(p, (apr_size_t)(len + 1)); // char *buf = cos_pcalloc(p, (apr_size_t)(len + 1));
char *buf = taosMemoryCalloc(1, (apr_size_t)(len)); char *buf = taosMemoryCalloc(1, (apr_size_t)(len));
if (!buf) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
// buf[len] = '\0'; // buf[len] = '\0';
cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) {
size = cos_buf_size(content); size = cos_buf_size(content);
@ -1609,12 +1613,13 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
} }
// cos_warn_log("Download data=%s", buf); // cos_warn_log("Download data=%s", buf);
_exit:
//销毁内存池 //销毁内存池
cos_pool_destroy(p); cos_pool_destroy(p);
*ppBlock = buf; *ppBlock = buf;
return code; TAOS_RETURN(code);
} }
typedef struct { typedef struct {

View File

@ -9,42 +9,35 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH); TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH);
if (!fd) { if (!fd) {
code = TAOS_SYSTEM_ERROR(errno); uError("%s Failed to open %s", __func__, cp_path);
uError("ERROR: %s Failed to open %s", __func__, cp_path); TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno));
return code;
} }
checkpoint->thefile = fd; checkpoint->thefile = fd;
return code; TAOS_RETURN(code);
} }
void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); } int32_t cos_cp_close(TdFilePtr fd) { return taosCloseFile(&fd); }
void cos_cp_remove(char const* filepath) { (void)taosRemoveFile(filepath); } int32_t cos_cp_remove(char const* filepath) { return taosRemoveFile(filepath); }
static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) { static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
int32_t code = 0; int32_t code = 0, lino = 0;
cJSON const* item2 = NULL; cJSON const* item2 = NULL;
cJSON* json = cJSON_Parse(cp_body); cJSON* json = cJSON_Parse(cp_body);
if (NULL == json) { if (NULL == json) {
code = TSDB_CODE_FILE_CORRUPTED; TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
} }
cJSON const* item = cJSON_GetObjectItem(json, "ver"); cJSON const* item = cJSON_GetObjectItem(json, "ver");
if (!cJSON_IsNumber(item) || item->valuedouble != 1) { if (!cJSON_IsNumber(item) || item->valuedouble != 1) {
code = TSDB_CODE_FILE_CORRUPTED; TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble);
goto _exit;
} }
item = cJSON_GetObjectItem(json, "type"); item = cJSON_GetObjectItem(json, "type");
if (!cJSON_IsNumber(item)) { if (!cJSON_IsNumber(item)) {
code = TSDB_CODE_FILE_CORRUPTED; TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
} }
cp->cp_type = item->valuedouble; cp->cp_type = item->valuedouble;
@ -155,40 +148,36 @@ static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
} }
_exit: _exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (json) cJSON_Delete(json); if (json) cJSON_Delete(json);
if (cp_body) taosMemoryFree(cp_body); if (cp_body) taosMemoryFree(cp_body);
TAOS_RETURN(code);
return code;
} }
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
int32_t code = 0; int32_t code = 0, lino = 0;
char* cp_body = NULL;
TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ); TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ);
if (!fd) { if (!fd) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
uError("ERROR: %s Failed to open %s", __func__, filepath);
goto _exit;
} }
int64_t size = -1; int64_t size = -1;
code = taosStatFile(filepath, &size, NULL, NULL); TAOS_CHECK_GOTO(taosStatFile(filepath, &size, NULL, NULL), &lino, _exit);
if (code) {
uError("ERROR: %s Failed to stat %s", __func__, filepath);
goto _exit;
}
char* cp_body = taosMemoryMalloc(size + 1); cp_body = taosMemoryMalloc(size + 1);
if (!cp_body) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
int64_t n = taosReadFile(fd, cp_body, size); int64_t n = taosReadFile(fd, cp_body, size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
uError("ERROR: %s Failed to read %s", __func__, filepath);
goto _exit;
} else if (n != size) { } else if (n != size) {
code = TSDB_CODE_FILE_CORRUPTED; TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size);
goto _exit;
} }
taosCloseFile(&fd); taosCloseFile(&fd);
cp_body[size] = '\0'; cp_body[size] = '\0';
@ -196,184 +185,161 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
return cos_cp_parse_body(cp_body, checkpoint); return cos_cp_parse_body(cp_body, checkpoint);
_exit: _exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (fd) { if (fd) {
taosCloseFile(&fd); taosCloseFile(&fd);
} }
if (cp_body) { if (cp_body) {
taosMemoryFree(cp_body); taosMemoryFree(cp_body);
} }
TAOS_RETURN(code);
return code;
} }
static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) { static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
int32_t code = 0; int32_t code = 0, lino = 0;
char* data = cJSON_PrintUnformatted(json); char* data = cJSON_PrintUnformatted(json);
if (NULL == data) { if (!data) {
return TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
} }
TdFilePtr fp = checkpoint->thefile; TdFilePtr fp = checkpoint->thefile;
if (taosFtruncateFile(fp, 0) < 0) { if (taosFtruncateFile(fp, 0) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto _exit;
} }
if (taosLSeekFile(fp, 0, SEEK_SET) < 0) { if (taosLSeekFile(fp, 0, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto _exit;
} }
if (taosWriteFile(fp, data, strlen(data)) < 0) { if (taosWriteFile(fp, data, strlen(data)) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto _exit;
} }
if (taosFsyncFile(fp) < 0) { if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto _exit;
} }
_exit: _exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosMemoryFree(data); taosMemoryFree(data);
return code; TAOS_RETURN(code);
} }
int32_t cos_cp_dump(SCheckpoint* cp) { int32_t cos_cp_dump(SCheckpoint* cp) {
int32_t code = 0; int32_t code = 0, lino = 0;
int32_t lino = 0;
cJSON* ojson = NULL; cJSON* ojson = NULL;
cJSON* json = cJSON_CreateObject(); cJSON* json = cJSON_CreateObject();
if (!json) return TSDB_CODE_OUT_OF_MEMORY; if (!json) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) { if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) { if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) { if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) { if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (COS_CP_TYPE_UPLOAD == cp->cp_type) { if (COS_CP_TYPE_UPLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "file"); ojson = cJSON_AddObjectToObject(json, "file");
if (!ojson) { if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) { if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) { if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) { if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) { if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) { } else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "object"); ojson = cJSON_AddObjectToObject(json, "object");
if (!ojson) { if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) { if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) { if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) { if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) { if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
ojson = cJSON_AddObjectToObject(json, "cpparts"); ojson = cJSON_AddObjectToObject(json, "cpparts");
if (!ojson) { if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) { if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) { if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts"); cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts");
if (!ajson) { if (!ajson) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
for (int i = 0; i < cp->part_num; ++i) { for (int i = 0; i < cp->part_num; ++i) {
cJSON* item = cJSON_CreateObject(); cJSON* item = cJSON_CreateObject();
if (!item) { if (!item) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
cJSON_AddItemToArray(ajson, item); cJSON_AddItemToArray(ajson, item);
if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) { if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) { if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) { if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) { if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) { if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) { if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = cos_cp_save_json(json, cp); code = cos_cp_save_json(json, cp);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(code, &lino, _exit);
_exit: _exit:
if (code) { if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
} }
cJSON_Delete(json); cJSON_Delete(json);
return code; TAOS_RETURN(code);
} }
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {} void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {}

View File

@ -615,6 +615,10 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex)
int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex; int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex;
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index);
if (pColInfoData == NULL) {
return 0;
}
if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) { if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) {
return 0; return 0;
} }
@ -1512,7 +1516,9 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
colDataDestroy(pColInfoData); colDataDestroy(pColInfoData);
} }
pBlock->pDataBlock = taosArrayDestroy(pBlock->pDataBlock); taosArrayDestroy(pBlock->pDataBlock);
pBlock->pDataBlock = NULL;
taosMemoryFreeClear(pBlock->pBlockAgg); taosMemoryFreeClear(pBlock->pBlockAgg);
memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
} }
@ -2486,7 +2492,10 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha
RandTableName rname = { RandTableName rname = {
.tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname};
buildChildTableName(&rname); int32_t code = buildChildTableName(&rname);
if(code != TSDB_CODE_SUCCESS){
return code;
}
taosArrayDestroy(tags); taosArrayDestroy(tags);
if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) {

View File

@ -305,11 +305,11 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo
*(int32_t *)(fixed + schema->columns[i].offset) = varlen - fixed - sinfo->tupleFixedSize; *(int32_t *)(fixed + schema->columns[i].offset) = varlen - fixed - sinfo->tupleFixedSize;
varlen += tPutU32v(varlen, colValArray[colValIndex].value.nData); varlen += tPutU32v(varlen, colValArray[colValIndex].value.nData);
if (colValArray[colValIndex].value.nData) { if (colValArray[colValIndex].value.nData) {
memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); (void)memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
varlen += colValArray[colValIndex].value.nData; varlen += colValArray[colValIndex].value.nData;
} }
} else { } else {
memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val, (void)memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val,
tDataTypes[schema->columns[i].type].bytes); tDataTypes[schema->columns[i].type].bytes);
} }
} else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL } else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL
@ -384,12 +384,12 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c
payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData); payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData);
if (colValArray[colValIndex].value.nData > 0) { if (colValArray[colValIndex].value.nData > 0) {
memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); (void)memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
} }
payloadSize += colValArray[colValIndex].value.nData; payloadSize += colValArray[colValIndex].value.nData;
} else { } else {
payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
memcpy(payload + payloadSize, &colValArray[colValIndex].value.val, (void)memcpy(payload + payloadSize, &colValArray[colValIndex].value.val,
tDataTypes[schema->columns[i].type].bytes); tDataTypes[schema->columns[i].type].bytes);
payloadSize += tDataTypes[schema->columns[i].type].bytes; payloadSize += tDataTypes[schema->columns[i].type].bytes;
} }
@ -475,7 +475,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted,
value.nData = infos[iInfo].bind->length[iRow]; value.nData = infos[iInfo].bind->length[iRow];
value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow; value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow;
} else { } else {
memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, (void)memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
infos[iInfo].bind->buffer_length); infos[iInfo].bind->buffer_length);
} }
colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); colVal = COL_VAL_VALUE(infos[iInfo].columnId, value);
@ -509,7 +509,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->cid = pTColumn->colId; pColVal->cid = pTColumn->colId;
pColVal->value.type = pTColumn->type; pColVal->value.type = pTColumn->type;
pColVal->flag = CV_FLAG_VALUE; pColVal->flag = CV_FLAG_VALUE;
memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); (void)memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY));
return 0; return 0;
} }
@ -573,7 +573,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = NULL; pColVal->value.pData = NULL;
} }
} else { } else {
memcpy(&pColVal->value.val, pData, pTColumn->bytes); (void)memcpy(&pColVal->value.val, pData, pTColumn->bytes);
} }
} }
return 0; return 0;
@ -624,7 +624,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset); pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset);
pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData); pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData);
} else { } else {
memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); (void)memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
} }
} }
@ -861,7 +861,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.cid = pTColumn->colId; pIter->cv.cid = pTColumn->colId;
pIter->cv.value.type = pTColumn->type; pIter->cv.value.type = pTColumn->type;
pIter->cv.flag = CV_FLAG_VALUE; pIter->cv.flag = CV_FLAG_VALUE;
memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); (void)memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY));
goto _exit; goto _exit;
} }
@ -906,7 +906,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL; pIter->cv.value.pData = NULL;
} }
} else { } else {
memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); (void)memcpy(&pIter->cv.value.val, pData, pTColumn->bytes);
} }
} }
@ -965,7 +965,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL; pIter->cv.value.pData = NULL;
} }
} else { } else {
memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); (void)memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
} }
goto _exit; goto _exit;
} }
@ -1288,7 +1288,7 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
key->pks[i].pData = tdata; key->pks[i].pData = tdata;
key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData); key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData);
} else { } else {
memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes); (void)memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes);
} }
} }
} }
@ -1378,7 +1378,7 @@ FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) {
return 0; return 0;
} }
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
pDst->ts = pSrc->ts; pDst->ts = pSrc->ts;
pDst->numOfPKs = pSrc->numOfPKs; pDst->numOfPKs = pSrc->numOfPKs;
@ -1392,12 +1392,10 @@ int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
} else { } else {
pVal->nData = pSrc->pks[i].nData; pVal->nData = pSrc->pks[i].nData;
ASSERT(pSrc->pks[i].pData != NULL); ASSERT(pSrc->pks[i].pData != NULL);
memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); (void)memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData);
} }
} }
} }
return TSDB_CODE_SUCCESS;
} }
// STag ======================================== // STag ========================================
@ -1528,7 +1526,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
} else { } else {
p = p ? p + n : p; p = p ? p + n : p;
n += tDataTypes[pTagVal->type].bytes; n += tDataTypes[pTagVal->type].bytes;
if (p) memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); if (p) (void)memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes);
} }
return n; return n;
@ -1550,7 +1548,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
if (IS_VAR_DATA_TYPE(pTagVal->type)) { if (IS_VAR_DATA_TYPE(pTagVal->type)) {
n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData);
} else { } else {
memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); (void)memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes);
n += tDataTypes[pTagVal->type].bytes; n += tDataTypes[pTagVal->type].bytes;
} }
@ -1661,7 +1659,7 @@ char *tTagValToData(const STagVal *value, bool isJson) {
} }
varDataLen(data + typeBytes) = value->nData; varDataLen(data + typeBytes) = value->nData;
memcpy(varDataVal(data + typeBytes), value->pData, value->nData); (void)memcpy(varDataVal(data + typeBytes), value->pData, value->nData);
} else { } else {
data = ((char *)&(value->i64)) - typeBytes; // json with type data = ((char *)&(value->i64)) - typeBytes; // json with type
} }
@ -1713,7 +1711,7 @@ bool tTagGet(const STag *pTag, STagVal *pTagVal) {
} else if (c > 0) { } else if (c > 0) {
lidx = midx + 1; lidx = midx + 1;
} else { } else {
memcpy(pTagVal, &tv, sizeof(tv)); (void)memcpy(pTagVal, &tv, sizeof(tv));
return true; return true;
} }
} }
@ -1892,7 +1890,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
if (nData) { if (nData) {
code = tRealloc(&pColData->pData, pColData->nData + nData); code = tRealloc(&pColData->pData, pColData->nData + nData);
if (code) goto _exit; if (code) goto _exit;
memcpy(pColData->pData + pColData->nData, pData, nData); (void)memcpy(pColData->pData + pColData->nData, pData, nData);
pColData->nData += nData; pColData->nData += nData;
} }
} else { } else {
@ -1900,7 +1898,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes);
if (code) goto _exit; if (code) goto _exit;
if (pData) { if (pData) {
memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); (void)memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]);
} else { } else {
memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]);
} }
@ -2594,7 +2592,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo
} }
value.pData = pColData->pData + pColData->aOffset[iVal]; value.pData = pColData->pData + pColData->aOffset[iVal];
} else { } else {
memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); (void)memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes);
} }
*pColVal = COL_VAL_VALUE(pColData->cid, value); *pColVal = COL_VAL_VALUE(pColData->cid, value);
} }
@ -2692,7 +2690,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal));
break; break;
case (HAS_VALUE | HAS_NULL | HAS_NONE): case (HAS_VALUE | HAS_NULL | HAS_NONE):
pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal)); pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal));
@ -2700,7 +2698,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal));
break; break;
default: default:
pColData->pBitMap = NULL; pColData->pBitMap = NULL;
@ -2714,7 +2712,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); (void)memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2);
} else { } else {
pColData->aOffset = NULL; pColData->aOffset = NULL;
} }
@ -2727,7 +2725,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
goto _exit; goto _exit;
} }
memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); (void)memcpy(pColData->pData, pColDataFrom->pData, pColData->nData);
} else { } else {
pColData->pData = NULL; pColData->pData = NULL;
} }
@ -3119,10 +3117,10 @@ static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SCo
pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData; pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData;
} }
memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], (void)memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow],
nData); nData);
} else { } else {
memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], (void)memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow],
&pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]); &pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]);
} }
return code; return code;
@ -3361,7 +3359,7 @@ static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd /
ASSERT(0); ASSERT(0);
} else { } else {
if (iv != iStart) { if (iv != iStart) {
memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], (void)memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart],
&pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]);
} }
memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)],
@ -3579,11 +3577,11 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) {
case (HAS_NULL | HAS_NONE): case (HAS_NULL | HAS_NONE):
case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NONE):
case (HAS_VALUE | HAS_NULL): case (HAS_VALUE | HAS_NULL):
if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal));
n += BIT1_SIZE(pColData->nVal); n += BIT1_SIZE(pColData->nVal);
break; break;
case (HAS_VALUE | HAS_NULL | HAS_NONE): case (HAS_VALUE | HAS_NULL | HAS_NONE):
if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal));
n += BIT2_SIZE(pColData->nVal); n += BIT2_SIZE(pColData->nVal);
break; break;
default: default:
@ -3593,14 +3591,14 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) {
// value // value
if (pColData->flag & HAS_VALUE) { if (pColData->flag & HAS_VALUE) {
if (IS_VAR_DATA_TYPE(pColData->type)) { if (IS_VAR_DATA_TYPE(pColData->type)) {
if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); if (pBuf) (void)memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2);
n += (pColData->nVal << 2); n += (pColData->nVal << 2);
n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData);
if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData);
n += pColData->nData; n += pColData->nData;
} else { } else {
if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData);
n += pColData->nData; n += pColData->nData;
} }
} }
@ -4348,7 +4346,7 @@ int32_t tCompressData(void *input, // input
ASSERT(outputSize >= extraSizeNeeded); ASSERT(outputSize >= extraSizeNeeded);
if (info->cmprAlg == NO_COMPRESSION) { if (info->cmprAlg == NO_COMPRESSION) {
memcpy(output, input, info->originalSize); (void)memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize; info->compressedSize = info->originalSize;
} else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local; SBuffer local;
@ -4385,7 +4383,7 @@ int32_t tCompressData(void *input, // input
} else { } else {
DEFINE_VAR(info->cmprAlg) DEFINE_VAR(info->cmprAlg)
if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) { if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) {
memcpy(output, input, info->originalSize); (void)memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize; info->compressedSize = info->originalSize;
return 0; return 0;
} }
@ -4431,7 +4429,7 @@ int32_t tDecompressData(void *input, // input
if (info->cmprAlg == NO_COMPRESSION) { if (info->cmprAlg == NO_COMPRESSION) {
ASSERT(info->compressedSize == info->originalSize); ASSERT(info->compressedSize == info->originalSize);
memcpy(output, input, info->compressedSize); (void)memcpy(output, input, info->compressedSize);
} else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local; SBuffer local;
@ -4468,7 +4466,7 @@ int32_t tDecompressData(void *input, // input
} else { } else {
DEFINE_VAR(info->cmprAlg); DEFINE_VAR(info->cmprAlg);
if (l1 == L1_DISABLED && l2 == L2_DISABLED) { if (l1 == L1_DISABLED && l2 == L2_DISABLED) {
memcpy(output, input, info->compressedSize); (void)memcpy(output, input, info->compressedSize);
return 0; return 0;
} }
SBuffer local; SBuffer local;

View File

@ -15,9 +15,9 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tmisce.h" #include "tmisce.h"
#include "tdatablock.h"
#include "tglobal.h" #include "tglobal.h"
#include "tjson.h" #include "tjson.h"
#include "tdatablock.h"
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = 0; pEp->port = 0;
@ -34,18 +34,26 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = tsServerPort; pEp->port = tsServerPort;
} }
if (pEp->port <= 0) {
return TSDB_CODE_INVALID_PARA;
}
return 0; return 0;
} }
void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) { int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) {
if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) { if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) {
return; return TSDB_CODE_INVALID_PARA;
} }
int32_t index = pEpSet->numOfEps; int32_t index = pEpSet->numOfEps;
if (index >= sizeof(pEpSet->eps) / sizeof(pEpSet->eps[0])) {
return TSDB_CODE_OUT_OF_RANGE;
}
tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn)); tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn));
pEpSet->eps[index].port = port; pEpSet->eps[index].port = port;
pEpSet->numOfEps += 1; pEpSet->numOfEps += 1;
return 0;
} }
bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) { bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) {
@ -131,75 +139,96 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet) {
return ep; return ep;
} }
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t bufLen) { int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) {
int len = snprintf(pBuf, bufLen, "epset:{"); int32_t ret = 0;
if (len < 0) { int32_t nwrite = 0;
return -1;
}
for (int _i = 0; (_i < pEpSet->numOfEps) && (bufLen > len); _i++) { nwrite = snprintf(pBuf + nwrite, cap, "epset:{");
if (nwrite <= 0 || nwrite >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
cap -= nwrite;
for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) {
int32_t ret = 0; int32_t ret = 0;
if (_i == pEpSet->numOfEps - 1) { if (_i == pEpSet->numOfEps - 1) {
ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} else { } else {
ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} }
if (ret < 0) { if (ret <= 0 || ret >= cap) {
return -1; return TSDB_CODE_OUT_OF_BUFFER;
} }
len += ret; nwrite += ret;
cap -= ret;
} }
if (len < bufLen) { if (cap <= 0) {
/*len += */snprintf(pBuf + len, bufLen - len, "}, inUse:%d", pEpSet->inUse); return TSDB_CODE_OUT_OF_BUFFER;
} }
ret = snprintf(pBuf + nwrite, cap, "}, inUse:%d", pEpSet->inUse);
if (ret <= 0 || ret >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) {
SJson* pJson = tjsonCreateObject(); int32_t code = 0;
if (pJson == NULL) return -1; SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
char tmp[4096] = {0}; char tmp[4096] = {0};
tjsonAddDoubleToObject(pJson, "reportVersion", 1); TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "reportVersion", 1), NULL, _exit);
tjsonAddIntegerToObject(pJson, "clusterId", clusterId); TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "clusterId", clusterId), NULL, _exit);
tjsonAddIntegerToObject(pJson, "startTime", startTime); TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "startTime", startTime), NULL, _exit);
// Do NOT invoke the taosGetFqdn here. // Do NOT invoke the taosGetFqdn here.
// this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked // this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked
// environment. The lock operation by taosGetFqdn may cause this program deadlock. // environment. The lock operation by taosGetFqdn may cause this program deadlock.
tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn), NULL, _exit);
tjsonAddIntegerToObject(pJson, "pid", taosGetPId()); TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "pid", taosGetPId()), NULL, _exit);
taosGetAppName(tmp, NULL); code = taosGetAppName(tmp, NULL);
tjsonAddStringToObject(pJson, "appName", tmp); if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "appName", tmp), NULL, _exit);
if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) { if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) {
tjsonAddStringToObject(pJson, "os", tmp); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "os", tmp), NULL, _exit);
} else {
// do nothing
} }
float numOfCores = 0; float numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) {
tjsonAddStringToObject(pJson, "cpuModel", tmp); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "cpuModel", tmp), NULL, _exit);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores), NULL, _exit);
} else { } else {
tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores); TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores), NULL, _exit);
} }
snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); int32_t nBytes = snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
tjsonAddStringToObject(pJson, "memory", tmp); if (nBytes <= 9 || nBytes >= sizeof(tmp)) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_RANGE, NULL, _exit);
}
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit);
tjsonAddStringToObject(pJson, "version", version); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", version), NULL, _exit);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", buildinfo), NULL, _exit);
tjsonAddStringToObject(pJson, "gitInfo", gitinfo);
tjsonAddIntegerToObject(pJson, "crashSig", signum); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", gitinfo), NULL, _exit);
tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs());
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit);
#ifdef _TD_DARWIN_64 #ifdef _TD_DARWIN_64
taosLogTraceToBuf(tmp, sizeof(tmp), 4); taosLogTraceToBuf(tmp, sizeof(tmp), 4);
@ -209,26 +238,47 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t
taosLogTraceToBuf(tmp, sizeof(tmp), 8); taosLogTraceToBuf(tmp, sizeof(tmp), 8);
#endif #endif
tjsonAddStringToObject(pJson, "stackInfo", tmp); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "stackInfo", tmp), NULL, _exit);
char* pCont = tjsonToString(pJson); char* pCont = tjsonToString(pJson);
if (pCont == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_GOTO(code, NULL, _exit);
goto _exit;
}
tjsonDelete(pJson); tjsonDelete(pJson);
*pMsg = pCont; *pMsg = pCont;
pJson = NULL;
return TSDB_CODE_SUCCESS; _exit:
tjsonDelete(pJson);
TAOS_RETURN(code);
} }
int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
SConfig* pConf = taosGetCfg(); int32_t code = 0;
SConfig* pConf = taosGetCfg();
if (pConf == NULL) {
return TSDB_CODE_INVALID_CFG;
}
int32_t numOfRows = 0; int32_t numOfRows = 0;
int32_t col = startCol; int32_t col = startCol;
SConfigItem* pItem = NULL; SConfigItem* pItem = NULL;
SConfigIter* pIter = NULL;
blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)); int8_t locked = 0;
SConfigIter* pIter = cfgCreateIter(pConf);
TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit);
pIter = cfgCreateIter(pConf);
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
cfgLock(pConf); cfgLock(pConf);
locked = 1;
while ((pItem = cfgNextIter(pIter)) != NULL) { while ((pItem = cfgNextIter(pIter)) != NULL) {
col = startCol; col = startCol;
@ -236,29 +286,44 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
// GRANT_CFG_SKIP; // GRANT_CFG_SKIP;
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, name, false); if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit);
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
int32_t valueLen = 0; int32_t valueLen = 0;
cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
varDataSetLen(value, valueLen); varDataSetLen(value, valueLen);
pColInfo = taosArrayGet(pBlock->pDataBlock, col++); pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, value, false); if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, value, false), NULL, _exit);
char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0};
cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen);
varDataSetLen(scope, valueLen); varDataSetLen(scope, valueLen);
pColInfo = taosArrayGet(pBlock->pDataBlock, col++); pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, scope, false); if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit);
numOfRows++; numOfRows++;
} }
cfgUnLock(pConf);
pBlock->info.rows = numOfRows; pBlock->info.rows = numOfRows;
_exit:
if (locked) cfgUnLock(pConf);
cfgDestroyIter(pIter); cfgDestroyIter(pIter);
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }

View File

@ -9505,7 +9505,8 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) {
static void tDeleteMqDataRspCommon(void *rsp) { static void tDeleteMqDataRspCommon(void *rsp) {
SMqDataRspCommon *pRsp = rsp; SMqDataRspCommon *pRsp = rsp;
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); taosArrayDestroy(pRsp->blockDataLen);
pRsp->blockDataLen = NULL;
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
pRsp->blockData = NULL; pRsp->blockData = NULL;
taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper);
@ -9558,7 +9559,8 @@ void tDeleteSTaosxRsp(void *rsp) {
tDeleteMqDataRspCommon(rsp); tDeleteMqDataRspCommon(rsp);
STaosxRsp *pRsp = (STaosxRsp *)rsp; STaosxRsp *pRsp = (STaosxRsp *)rsp;
pRsp->createTableLen = taosArrayDestroy(pRsp->createTableLen); taosArrayDestroy(pRsp->createTableLen);
pRsp->createTableLen = NULL;
taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree);
pRsp->createTableReq = NULL; pRsp->createTableReq = NULL;
} }

View File

@ -293,17 +293,19 @@ static int compareKv(const void* p1, const void* p2) {
/* /*
* use stable name and tags to grearate child table name * use stable name and tags to grearate child table name
*/ */
void buildChildTableName(RandTableName* rName) { int32_t buildChildTableName(RandTableName* rName) {
SStringBuilder sb = {0}; SStringBuilder sb = {0};
taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen);
if (sb.buf == NULL) { if (sb.buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return;
} }
taosArraySort(rName->tags, compareKv); taosArraySort(rName->tags, compareKv);
for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) {
taosStringBuilderAppendChar(&sb, ','); taosStringBuilderAppendChar(&sb, ',');
SSmlKv* tagKv = taosArrayGet(rName->tags, j); SSmlKv* tagKv = taosArrayGet(rName->tags, j);
if(tagKv == NULL) {
return TSDB_CODE_SML_INVALID_DATA;
}
taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
taosStringBuilderAppendChar(&sb, '='); taosStringBuilderAppendChar(&sb, '=');
if (IS_VAR_DATA_TYPE(tagKv->type)) { if (IS_VAR_DATA_TYPE(tagKv->type)) {
@ -323,8 +325,9 @@ void buildChildTableName(RandTableName* rName) {
rName->ctbShortName[0] = 't'; rName->ctbShortName[0] = 't';
rName->ctbShortName[1] = '_'; rName->ctbShortName[1] = '_';
for (int i = 0; i < 16; i++) { for (int i = 0; i < 16; i++) {
sprintf(temp, "%02x", context.digest[i]); (void)sprintf(temp, "%02x", context.digest[i]);
strcat(rName->ctbShortName, temp); (void)strcat(rName->ctbShortName, temp);
} }
taosStringBuilderDestroy(&sb); taosStringBuilderDestroy(&sb);
return TSDB_CODE_SUCCESS;
} }

View File

@ -34,12 +34,14 @@ typedef struct SVnodeMgmt {
SAutoQWorkerPool streamPool; SAutoQWorkerPool streamPool;
SWWorkerPool fetchPool; SWWorkerPool fetchPool;
SSingleWorker mgmtWorker; SSingleWorker mgmtWorker;
SSingleWorker mgmtMultiWorker;
SHashObj *hash; SHashObj *hash;
TdThreadRwlock lock; TdThreadRwlock lock;
SVnodesStat state; SVnodesStat state;
STfs *pTfs; STfs *pTfs;
TdThread thread; TdThread thread;
bool stop; bool stop;
TdThreadMutex createLock;
} SVnodeMgmt; } SVnodeMgmt;
typedef struct { typedef struct {
@ -69,6 +71,7 @@ typedef struct {
STaosQueue *pQueryQ; STaosQueue *pQueryQ;
STaosQueue *pStreamQ; STaosQueue *pStreamQ;
STaosQueue *pFetchQ; STaosQueue *pFetchQ;
STaosQueue *pMultiMgmQ;
} SVnodeObj; } SVnodeObj;
typedef struct { typedef struct {
@ -125,6 +128,7 @@ int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -377,11 +377,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
goto _OVER; goto _OVER;
} }
taosThreadMutexLock(&pMgmt->createLock);
code = vmWriteVnodeListToFile(pMgmt); code = vmWriteVnodeListToFile(pMgmt);
if (code != 0) { if (code != 0) {
code = terrno != 0 ? terrno : code; code = terrno != 0 ? terrno : code;
taosThreadMutexUnlock(&pMgmt->createLock);
goto _OVER; goto _OVER;
} }
taosThreadMutexUnlock(&pMgmt->createLock);
_OVER: _OVER:
if (code != 0) { if (code != 0) {
@ -985,7 +988,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMultiMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

View File

@ -439,6 +439,8 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
dInfo("start to close all vnodes"); dInfo("start to close all vnodes");
tSingleWorkerCleanup(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->mgmtWorker);
dInfo("vnodes mgmt worker is stopped"); dInfo("vnodes mgmt worker is stopped");
tSingleWorkerCleanup(&pMgmt->mgmtMultiWorker);
dInfo("vnodes multiple mgmt worker is stopped");
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
@ -506,6 +508,7 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
vmStopWorker(pMgmt); vmStopWorker(pMgmt);
vnodeCleanup(); vnodeCleanup();
taosThreadRwlockDestroy(&pMgmt->lock); taosThreadRwlockDestroy(&pMgmt->lock);
taosThreadMutexDestroy(&pMgmt->createLock);
taosMemoryFree(pMgmt); taosMemoryFree(pMgmt);
} }
@ -580,6 +583,7 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize; pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize;
pMgmt->msgCb.mgmt = pMgmt; pMgmt->msgCb.mgmt = pMgmt;
taosThreadRwlockInit(&pMgmt->lock, NULL); taosThreadRwlockInit(&pMgmt->lock, NULL);
taosThreadMutexInit(&pMgmt->createLock, NULL);
pMgmt->pTfs = pInput->pTfs; pMgmt->pTfs = pInput->pTfs;
if (pMgmt->pTfs == NULL) { if (pMgmt->pTfs == NULL) {

View File

@ -28,6 +28,31 @@ static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) {
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
static void vmProcessMultiMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SVnodeMgmt *pMgmt = pInfo->ahandle;
int32_t code = -1;
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, get from vnode-multi-mgmt queue", pMsg);
switch (pMsg->msgType) {
case TDMT_DND_CREATE_VNODE:
code = vmProcessCreateVnodeReq(pMgmt, pMsg);
break;
}
if (IsReq(pMsg)) {
if (code != 0) {
if (terrno != 0) code = terrno;
dGError("msg:%p, failed to process since %s, type:%s", pMsg, tstrerror(code), TMSG_INFO(pMsg->msgType));
}
vmSendRsp(pMsg, code);
}
dGTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SVnodeMgmt *pMgmt = pInfo->ahandle; SVnodeMgmt *pMgmt = pInfo->ahandle;
int32_t code = -1; int32_t code = -1;
@ -271,6 +296,13 @@ int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsg
int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, STREAM_QUEUE); } int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, STREAM_QUEUE); }
int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-multi-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg);
return 0;
}
int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-mgmt queue", pMsg); dGTrace("msg:%p, put into vnode-mgmt queue", pMsg);
@ -415,6 +447,20 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) return -1; if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) return -1;
int32_t threadNum = 0;
if (tsNumOfCores == 1) {
threadNum = 2;
} else {
threadNum = tsNumOfCores;
}
SSingleWorkerCfg multiMgmtCfg = {.min = threadNum,
.max = threadNum,
.name = "vnode-multi-mgmt",
.fp = (FItem)vmProcessMultiMgmtQueue,
.param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg) != 0) return -1;
dDebug("vnode workers are initialized"); dDebug("vnode workers are initialized");
return 0; return 0;
} }

View File

@ -18,6 +18,7 @@
#include "audit.h" #include "audit.h"
#include "libs/function/tudf.h" #include "libs/function/tudf.h"
#include "tgrant.h" #include "tgrant.h"
#include "tcompare.h"
#define DM_INIT_AUDIT() \ #define DM_INIT_AUDIT() \
do { \ do { \
@ -163,6 +164,7 @@ int32_t dmInit() {
if (dmInitMonitor() != 0) return -1; if (dmInitMonitor() != 0) return -1;
if (dmInitAudit() != 0) return -1; if (dmInitAudit() != 0) return -1;
if (dmInitDnode(dmInstance()) != 0) return -1; if (dmInitDnode(dmInstance()) != 0) return -1;
if (InitRegexCache() != 0) return -1;
#if defined(USE_S3) #if defined(USE_S3)
if (s3Begin() != 0) return -1; if (s3Begin() != 0) return -1;
#endif #endif
@ -192,6 +194,7 @@ void dmCleanup() {
udfStopUdfd(); udfStopUdfd();
taosStopCacheRefreshWorker(); taosStopCacheRefreshWorker();
dmDiskClose(); dmDiskClose();
DestroyRegexCache();
#if defined(USE_S3) #if defined(USE_S3)
s3End(); s3End();
@ -310,7 +313,7 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype]; pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
dmReleaseWrapper(pWrapper); taosThreadMutexUnlock(&pDnode->mutex);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
return -1; return -1;

View File

@ -195,7 +195,9 @@ void *freeStreamTasks(SArray *pTaskLevel) {
taosArrayDestroy(pLevel); taosArrayDestroy(pLevel);
} }
return taosArrayDestroy(pTaskLevel); taosArrayDestroy(pTaskLevel);
return NULL;
} }
void tFreeStreamObj(SStreamObj *pStream) { void tFreeStreamObj(SStreamObj *pStream) {

View File

@ -978,7 +978,8 @@ void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpo
} }
void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) { void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
pInfo->pTaskList = taosArrayDestroy(pInfo->pTaskList); taosArrayDestroy(pInfo->pTaskList);
pInfo->pTaskList = NULL;
} }
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) { int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) {

View File

@ -167,7 +167,7 @@ void tsdbReaderClose2(STsdbReader *pReader);
int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext);
int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA);
void tsdbReleaseDataBlock2(STsdbReader *pReader); void tsdbReleaseDataBlock2(STsdbReader *pReader);
SSDataBlock *tsdbRetrieveDataBlock2(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList);
int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond);
int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle); int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle);

View File

@ -909,9 +909,9 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree);
bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree);
SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo);
void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void * destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
// tsdbCache ============================================================================================== // tsdbCache ==============================================================================================
typedef enum { typedef enum {

View File

@ -54,7 +54,6 @@ extern "C" {
#endif #endif
typedef struct SVnodeInfo SVnodeInfo; typedef struct SVnodeInfo SVnodeInfo;
typedef struct SMeta SMeta;
typedef struct SSma SSma; typedef struct SSma SSma;
typedef struct STsdb STsdb; typedef struct STsdb STsdb;
typedef struct STQ STQ; typedef struct STQ STQ;
@ -153,7 +152,6 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode);
void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive); void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive);
// meta // meta
typedef struct SMCtbCursor SMCtbCursor;
typedef struct SMStbCursor SMStbCursor; typedef struct SMStbCursor SMStbCursor;
typedef struct STbUidStore STbUidStore; typedef struct STbUidStore STbUidStore;

View File

@ -250,7 +250,8 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
} }
tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
tagArray = taosArrayDestroy(tagArray); taosArrayDestroy(tagArray);
tagArray = NULL;
if (pCreateTbReq->ctb.pTag == NULL) { if (pCreateTbReq->ctb.pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq); tdDestroySVCreateTbReq(pCreateTbReq);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -514,7 +515,8 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
pTableData->aRowP = taosArrayInit(numOfRows, sizeof(SRow*)); pTableData->aRowP = taosArrayInit(numOfRows, sizeof(SRow*));
if (pTableData->aRowP == NULL || pVals == NULL) { if (pTableData->aRowP == NULL || pVals == NULL) {
pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pTableData->aRowP);
pTableData->aRowP = NULL;
taosArrayDestroy(pVals); taosArrayDestroy(pVals);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
tqError("s-task:%s failed to prepare write stream res blocks, code:%s", id, tstrerror(code)); tqError("s-task:%s failed to prepare write stream res blocks, code:%s", id, tstrerror(code));
@ -539,7 +541,8 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
if (ts < earlyTs) { if (ts < earlyTs) {
tqError("s-task:%s ts:%" PRId64 " of generated results out of valid time range %" PRId64 " , discarded", id, tqError("s-task:%s ts:%" PRId64 " of generated results out of valid time range %" PRId64 " , discarded", id,
ts, earlyTs); ts, earlyTs);
pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); taosArrayDestroy(pTableData->aRowP);
pTableData->aRowP = NULL;
taosArrayDestroy(pVals); taosArrayDestroy(pVals);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

View File

@ -54,19 +54,23 @@ static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) {
if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
if (pRsp->blockData != NULL) { if (pRsp->blockData != NULL) {
pRsp->blockData = taosArrayDestroy(pRsp->blockData); taosArrayDestroy(pRsp->blockData);
pRsp->blockData = NULL;
} }
if (pRsp->blockDataLen != NULL) { if (pRsp->blockDataLen != NULL) {
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); taosArrayDestroy(pRsp->blockDataLen);
pRsp->blockDataLen = NULL;
} }
if (pRsp->blockTbName != NULL) { if (pRsp->blockTbName != NULL) {
pRsp->blockTbName = taosArrayDestroy(pRsp->blockTbName); taosArrayDestroy(pRsp->blockTbName);
pRsp->blockTbName = NULL;
} }
if (pRsp->blockSchema != NULL) { if (pRsp->blockSchema != NULL) {
pRsp->blockSchema = taosArrayDestroy(pRsp->blockSchema); taosArrayDestroy(pRsp->blockSchema);
pRsp->blockSchema = NULL;
} }
return -1; return -1;
} }

View File

@ -2579,7 +2579,7 @@ typedef struct {
static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) { tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) {
int32_t code = 0; int32_t code = 0;
pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, NULL); destroySttBlockReader(pr->pLDataIterArray, NULL);
pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
SMergeTreeConf conf = { SMergeTreeConf conf = {
@ -3187,7 +3187,8 @@ static void clearLastFileSet(SFSNextRowIter *state) {
int32_t iter = 0; int32_t iter = 0;
while ((pe = tSimpleHashIterate(state->pr->pTableMap, pe, &iter)) != NULL) { while ((pe = tSimpleHashIterate(state->pr->pTableMap, pe, &iter)) != NULL) {
STableLoadInfo *pInfo = *(STableLoadInfo **)pe; STableLoadInfo *pInfo = *(STableLoadInfo **)pe;
pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); taosArrayDestroy(pInfo->pTombData);
pInfo->pTombData = NULL;
} }
} }
} }
@ -3211,7 +3212,10 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
pIter->pMemDelData = NULL; pIter->pMemDelData = NULL;
loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); code = loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; pIter->idx = (SBlockIdx){.suid = suid, .uid = uid};

View File

@ -203,7 +203,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf
pReader->pTableList = pTableIdList; pReader->pTableList = pTableIdList;
pReader->numOfTables = numOfTables; pReader->numOfTables = numOfTables;
pReader->lastTs = INT64_MIN; pReader->lastTs = INT64_MIN;
pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, NULL); destroySttBlockReader(pReader->pLDataIterArray, NULL);
pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -311,7 +311,8 @@ void* tsdbCacherowsReaderClose(void* pReader) {
int32_t iter = 0; int32_t iter = 0;
while ((pe = tSimpleHashIterate(p->pTableMap, pe, &iter)) != NULL) { while ((pe = tSimpleHashIterate(p->pTableMap, pe, &iter)) != NULL) {
STableLoadInfo* pInfo = *(STableLoadInfo**)pe; STableLoadInfo* pInfo = *(STableLoadInfo**)pe;
pInfo->pTombData = taosArrayDestroy(pInfo->pTombData); taosArrayDestroy(pInfo->pTombData);
pInfo->pTombData = NULL;
} }
tSimpleHashCleanup(p->pTableMap); tSimpleHashCleanup(p->pTableMap);

View File

@ -23,11 +23,12 @@
static void tLDataIterClose2(SLDataIter *pIter); static void tLDataIterClose2(SLDataIter *pIter);
// SLDataIter ================================================= // SLDataIter =================================================
SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) { int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo) {
*pInfo = NULL;
SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo)); SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo));
if (pLoadInfo == NULL) { if (pLoadInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
pLoadInfo->blockData[0].sttBlockIndex = -1; pLoadInfo->blockData[0].sttBlockIndex = -1;
@ -37,26 +38,29 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList,
int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data); int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data);
if (code) { if (code) {
terrno = code; taosMemoryFreeClear(pLoadInfo);
return code;
} }
code = tBlockDataCreate(&pLoadInfo->blockData[1].data); code = tBlockDataCreate(&pLoadInfo->blockData[1].data);
if (code) { if (code) {
terrno = code; taosMemoryFreeClear(pLoadInfo);
return code;
} }
pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk));
if (pLoadInfo->aSttBlk == NULL) { if (pLoadInfo->aSttBlk == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFreeClear(pLoadInfo); taosMemoryFreeClear(pLoadInfo);
return NULL; return code;
} }
pLoadInfo->pSchema = pSchema; pLoadInfo->pSchema = pSchema;
pLoadInfo->colIds = colList; pLoadInfo->colIds = colList;
pLoadInfo->numOfCols = numOfCols; pLoadInfo->numOfCols = numOfCols;
return pLoadInfo; *pInfo = pLoadInfo;
return code;
} }
static void freeItem(void* pValue) { static void freeItem(void* pValue) {
@ -66,9 +70,9 @@ static void freeItem(void* pValue) {
} }
} }
void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
if (pLoadInfo == NULL) { if (pLoadInfo == NULL) {
return NULL; return;
} }
pLoadInfo->currentLoadBlockIndex = 1; pLoadInfo->currentLoadBlockIndex = 1;
@ -94,7 +98,6 @@ void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
taosArrayDestroy(pLoadInfo->aSttBlk); taosArrayDestroy(pLoadInfo->aSttBlk);
taosMemoryFree(pLoadInfo); taosMemoryFree(pLoadInfo);
return NULL;
} }
void destroyLDataIter(SLDataIter *pIter) { void destroyLDataIter(SLDataIter *pIter) {
@ -103,9 +106,9 @@ void destroyLDataIter(SLDataIter *pIter) {
taosMemoryFree(pIter); taosMemoryFree(pIter);
} }
void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) { void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) {
if (pLDataIterArray == NULL) { if (pLDataIterArray == NULL) {
return NULL; return;
} }
int32_t numOfLevel = taosArrayGetSize(pLDataIterArray); int32_t numOfLevel = taosArrayGetSize(pLDataIterArray);
@ -132,7 +135,6 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa
} }
taosArrayDestroy(pLDataIterArray); taosArrayDestroy(pLDataIterArray);
return NULL;
} }
// choose the unpinned slot to load next data block // choose the unpinned slot to load next data block
@ -914,9 +916,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF
} }
if (pLoadInfo == NULL) { if (pLoadInfo == NULL) {
pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pLoadInfo);
if (pLoadInfo == NULL) { if (code != TSDB_CODE_SUCCESS) {
code = terrno;
goto _end; goto _end;
} }
} }

File diff suppressed because it is too large Load Diff

View File

@ -112,22 +112,28 @@ void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {
taosArrayDestroy(pBuf->pData); taosArrayDestroy(pBuf->pData);
} }
void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) {
*pInfo = NULL;
int32_t bucketIndex = index / pBuf->numPerBucket; int32_t bucketIndex = index / pBuf->numPerBucket;
char** pBucket = taosArrayGet(pBuf->pData, bucketIndex); char** pBucket = taosArrayGet(pBuf->pData, bucketIndex);
return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo); if (pBucket == NULL) {
} return TSDB_CODE_FAILED;
STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) {
STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (p == NULL || *p == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
int32_t size = tSimpleHashGetSize(pTableMap);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);
return NULL;
} }
return *p; *pInfo = (STableBlockScanInfo*)((*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo));
return TSDB_CODE_SUCCESS;
}
int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) {
*pInfo = *(STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (pInfo == NULL) {
int32_t size = tSimpleHashGetSize(pTableMap);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);
return TSDB_CODE_INVALID_PARA;
}
return TSDB_CODE_SUCCESS;
} }
int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) { int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) {
@ -146,17 +152,17 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
} }
case TSDB_DATA_TYPE_INT:{ case TSDB_DATA_TYPE_INT:{
int32_t min = INT32_MIN; int32_t min = INT32_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break; break;
} }
case TSDB_DATA_TYPE_SMALLINT:{ case TSDB_DATA_TYPE_SMALLINT:{
int16_t min = INT16_MIN; int16_t min = INT16_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break; break;
} }
case TSDB_DATA_TYPE_TINYINT:{ case TSDB_DATA_TYPE_TINYINT:{
int8_t min = INT8_MIN; int8_t min = INT8_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break; break;
} }
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
@ -245,20 +251,27 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH
initLastProcKey(pScanInfo, pReader); initLastProcKey(pScanInfo, pReader);
pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); int32_t code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid, tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid,
pScanInfo->lastProcKey.ts, pReader->idStr); pScanInfo->lastProcKey.ts, pReader->idStr);
return TSDB_CODE_SUCCESS; return code;
} }
// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model
SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables) { STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) {
int32_t code = 0;
*pHashObj = NULL;
// allocate buffer in order to load data blocks from file // allocate buffer in order to load data blocks from file
// todo use simple hash instead, optimize the memory consumption // todo use simple hash instead, optimize the memory consumption
SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (pTableMap == NULL) { if (pTableMap == NULL) {
return NULL; return terrno;
} }
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
@ -267,7 +280,7 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
if (pUidList->tableUidList == NULL) { if (pUidList->tableUidList == NULL) {
tSimpleHashCleanup(pTableMap); tSimpleHashCleanup(pTableMap);
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pUidList->currentIndex = 0; pUidList->currentIndex = 0;
@ -275,8 +288,16 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
for (int32_t j = 0; j < numOfTables; ++j) { for (int32_t j = 0; j < numOfTables; ++j) {
pUidList->tableUidList[j] = idList[j].uid; pUidList->tableUidList[j] = idList[j].uid;
STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j); STableBlockScanInfo* pScanInfo = NULL;
initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader); code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo);
if (code != TSDB_CODE_SUCCESS) {
break;
}
code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);
if (code != TSDB_CODE_SUCCESS) {
break;
}
} }
taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc); taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc);
@ -286,7 +307,8 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
(sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList, (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList,
pTsdbReader->idStr); pTsdbReader->idStr);
return pTableMap; *pHashObj = pTableMap;
return code;
} }
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {
@ -308,7 +330,8 @@ void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {
pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter);
} }
pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); taosArrayDestroy(pInfo->delSkyline);
pInfo->delSkyline = NULL;
pInfo->lastProcKey.ts = ts; pInfo->lastProcKey.ts = ts;
// todo check the nextProcKey info // todo check the nextProcKey info
pInfo->sttKeyInfo.nextProcKey.ts = ts + step; pInfo->sttKeyInfo.nextProcKey.ts = ts + step;
@ -329,11 +352,16 @@ void clearBlockScanInfo(STableBlockScanInfo* p) {
p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter); p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter);
} }
p->delSkyline = taosArrayDestroy(p->delSkyline); taosArrayDestroy(p->delSkyline);
p->pBlockList = taosArrayDestroy(p->pBlockList); p->delSkyline = NULL;
p->pBlockIdxList = taosArrayDestroy(p->pBlockIdxList); taosArrayDestroy(p->pBlockList);
p->pMemDelData = taosArrayDestroy(p->pMemDelData); p->pBlockList = NULL;
p->pFileDelData = taosArrayDestroy(p->pFileDelData); taosArrayDestroy(p->pBlockIdxList);
p->pBlockIdxList = NULL;
taosArrayDestroy(p->pMemDelData);
p->pMemDelData = NULL;
taosArrayDestroy(p->pFileDelData);
p->pFileDelData = NULL;
clearRowKey(&p->lastProcKey); clearRowKey(&p->lastProcKey);
clearRowKey(&p->sttRange.skey); clearRowKey(&p->sttRange.skey);
@ -385,11 +413,13 @@ void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray
pIter->pBrinBlockList = pList; pIter->pBrinBlockList = pList;
} }
SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {
*pRecord = NULL;
if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) { if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) {
pIter->blockIndex += 1; pIter->blockIndex += 1;
if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) {
return NULL; return TSDB_CODE_FAILED;
} }
pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);
@ -398,7 +428,7 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));
return NULL; return TSDB_CODE_FAILED;
} }
pIter->recordIndex = -1; pIter->recordIndex = -1;
@ -406,7 +436,9 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
pIter->recordIndex += 1; pIter->recordIndex += 1;
tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record);
return &pIter->record; *pRecord = &pIter->record;
return TSDB_CODE_SUCCESS;
} }
void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); }
@ -579,7 +611,8 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
} }
taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList);
pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList); taosArrayDestroy(pTableScanInfo->pBlockList);
pTableScanInfo->pBlockList = NULL;
int64_t et = taosGetTimestampUs(); int64_t et = taosGetTimestampUs();
tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s",
@ -624,7 +657,8 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
for (int32_t i = 0; i < numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i);
pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList); taosArrayDestroy(pTableScanInfo->pBlockList);
pTableScanInfo->pBlockList = NULL;
} }
int64_t et = taosGetTimestampUs(); int64_t et = taosGetTimestampUs();
@ -662,7 +696,13 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
STombRecord record = {0}; STombRecord record = {0};
uint64_t uid = pReader->status.uidList.tableUidList[*j]; uint64_t uid = pReader->status.uidList.tableUidList[*j];
STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); STableBlockScanInfo* pScanInfo = NULL;
code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pScanInfo->pFileDelData == NULL) { if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
} }
@ -694,7 +734,11 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
} }
uid = pReader->status.uidList.tableUidList[*j]; uid = pReader->status.uidList.tableUidList[*j];
pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pScanInfo->pFileDelData == NULL) { if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
} }
@ -798,9 +842,12 @@ int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileRead
return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false); return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false);
} }
void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {
if (*ppMemDelData == NULL) { if (*ppMemDelData == NULL) {
*ppMemDelData = taosArrayInit(4, sizeof(SDelData)); *ppMemDelData = taosArrayInit(4, sizeof(SDelData));
if (*ppMemDelData == NULL) {
return TSDB_CODE_SUCCESS;
}
} }
SArray* pMemDelData = *ppMemDelData; SArray* pMemDelData = *ppMemDelData;
@ -828,6 +875,8 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT
p = p->pNext; p = p->pNext;
} }
} }
return TSDB_CODE_SUCCESS;
} }
int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
@ -952,6 +1001,7 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet)
int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf, int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf,
const char* pstr) { const char* pstr) {
int32_t numOfRows = 0; int32_t numOfRows = 0;
int32_t code = 0;
// no data exists, go to end // no data exists, go to end
int32_t numOfLevels = pFileSet->lvlArr->size; int32_t numOfLevels = pFileSet->lvlArr->size;
@ -976,7 +1026,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
conf.file[0] = *pSttLevel->fobjArr->data[i]->f; conf.file[0] = *pSttLevel->fobjArr->data[i]->f;
const char* pName = pSttLevel->fobjArr->data[i]->fname; const char* pName = pSttLevel->fobjArr->data[i]->fname;
int32_t code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader); code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr); tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr);
continue; continue;
@ -984,8 +1034,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
} }
if (pIter->pBlockLoadInfo == NULL) { if (pIter->pBlockLoadInfo == NULL) {
pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo);
if (pIter->pBlockLoadInfo == NULL) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to create block load info, code: out of memory, %s", pstr); tsdbError("failed to create block load info, code: out of memory, %s", pstr);
continue; continue;
} }
@ -993,7 +1043,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
// load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file
TStatisBlkArray* pStatisBlkArray = NULL; TStatisBlkArray* pStatisBlkArray = NULL;
int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr); tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr);
continue; continue;

View File

@ -315,32 +315,31 @@ typedef struct SBrinRecordIter {
} SBrinRecordIter; } SBrinRecordIter;
int32_t uidComparFunc(const void* p1, const void* p2); int32_t uidComparFunc(const void* p1, const void* p2);
int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id);
STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj);
SSHashObj* createDataBlockScanInfo(STsdbReader* pReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,
STableUidList* pUidList, int32_t numOfTables); STsdbReader* pReader);
int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, void clearBlockScanInfo(STableBlockScanInfo* p);
STsdbReader* pReader); void destroyAllBlockScanInfo(SSHashObj* pTableMap);
void clearBlockScanInfo(STableBlockScanInfo* p); void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step);
void destroyAllBlockScanInfo(SSHashObj* pTableMap); void cleanupInfoForNextFileset(SSHashObj* pTableMap);
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables);
void cleanupInfoForNextFileset(SSHashObj* pTableMap); void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf);
int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pRes);
void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf);
void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index);
// brin records iterator // brin records iterator
void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList);
SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter); int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord);
void clearBrinBlockIter(SBrinRecordIter* pIter); void clearBrinBlockIter(SBrinRecordIter* pIter);
// initialize block iterator API // initialize block iterator API
int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList); int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList);
bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr); bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr);
// load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time) // load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time)
void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver);
int32_t loadDataFileTombDataForAll(STsdbReader* pReader); int32_t loadDataFileTombDataForAll(STsdbReader* pReader);
int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo); int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo);
int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
@ -358,10 +357,10 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2);
int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc); int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc);
void clearRowKey(SRowKey* pKey); void clearRowKey(SRowKey* pKey);
bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp); bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp);
void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk); int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk);
void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree); void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree);
void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk); void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk);
typedef struct { typedef struct {
SArray* pTombData; SArray* pTombData;

View File

@ -32,8 +32,8 @@ char* tsAuditBatchUri = "/audit-batch";
int32_t auditInit(const SAuditCfg *pCfg) { int32_t auditInit(const SAuditCfg *pCfg) {
tsAudit.cfg = *pCfg; tsAudit.cfg = *pCfg;
tsAudit.records = taosArrayInit(0, sizeof(SAuditRecord *)); tsAudit.records = taosArrayInit(0, sizeof(SAuditRecord *));
taosThreadMutexInit(&tsAudit.lock, NULL); if(tsAudit.records == NULL) return TSDB_CODE_OUT_OF_MEMORY;
return 0; return taosThreadMutexInit(&tsAudit.lock, NULL);
} }
static FORCE_INLINE void auditDeleteRecord(SAuditRecord * record) { static FORCE_INLINE void auditDeleteRecord(SAuditRecord * record) {

View File

@ -929,7 +929,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i
SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore,
SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr,
SStorageAPI* pApi, int32_t tsIndex); SStorageAPI* pApi, int32_t tsIndex);
void initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic);
void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins);
void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);

View File

@ -20,6 +20,7 @@ extern "C" {
#endif #endif
#include "executorInt.h" #include "executorInt.h"
#include "tutil.h"
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type); void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo); bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);

View File

@ -96,7 +96,8 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
pGroupResInfo->freeItem = false; pGroupResInfo->freeItem = false;
pGroupResInfo->pRows = NULL; pGroupResInfo->pRows = NULL;
} else { } else {
pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); taosArrayDestroy(pGroupResInfo->pRows);
pGroupResInfo->pRows = NULL;
} }
pGroupResInfo->index = 0; pGroupResInfo->index = 0;
} }
@ -2102,7 +2103,8 @@ void* tableListDestroy(STableListInfo* pTableListInfo) {
return NULL; return NULL;
} }
pTableListInfo->pTableList = taosArrayDestroy(pTableListInfo->pTableList); taosArrayDestroy(pTableListInfo->pTableList);
pTableListInfo->pTableList = NULL;
taosMemoryFreeClear(pTableListInfo->groupOffset); taosMemoryFreeClear(pTableListInfo->groupOffset);
taosHashCleanup(pTableListInfo->map); taosHashCleanup(pTableListInfo->map);

View File

@ -479,12 +479,14 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
} }
int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
int32_t* tversion, int32_t idx) { int32_t* tversion, int32_t idx, bool* tbGet) {
*tbGet = false;
ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL); ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL);
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
if (taosArrayGetSize(pTaskInfo->schemaInfos) <= idx) { if (taosArrayGetSize(pTaskInfo->schemaInfos) <= idx) {
return -1; return TSDB_CODE_SUCCESS;
} }
SSchemaInfo* pSchemaInfo = taosArrayGet(pTaskInfo->schemaInfos, idx); SSchemaInfo* pSchemaInfo = taosArrayGet(pTaskInfo->schemaInfos, idx);
@ -502,6 +504,8 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
tableName[0] = 0; tableName[0] = 0;
} }
*tbGet = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

View File

@ -17,6 +17,7 @@
#include "function.h" #include "function.h"
#include "os.h" #include "os.h"
#include "tname.h" #include "tname.h"
#include "tutil.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tmsg.h" #include "tmsg.h"
@ -1357,11 +1358,13 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI;
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
return; return code;
} }
SStreamScanInfo* pScanInfo = downstream->info; SStreamScanInfo* pScanInfo = downstream->info;
@ -1369,8 +1372,9 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup
pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartScalarSup = pExpr;
pScanInfo->pPartTbnameSup = pTbnameExpr; pScanInfo->pPartTbnameSup = pTbnameExpr;
if (!pScanInfo->pUpdateInfo) { if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
} }
return code;
} }
SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) {
@ -1421,6 +1425,7 @@ void freePartItem(void* ptr) {
SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo) { SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -1511,14 +1516,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
return pOperator; return pOperator;
_error: _error:
pTaskInfo->code = code; pTaskInfo->code = code;
destroyStreamPartitionOperatorInfo(pInfo); destroyStreamPartitionOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }

View File

@ -388,9 +388,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost->totalCheckedRows += pBlock->info.rows; pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1; pCost->loadBlocks += 1;
SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); SSDataBlock* p = NULL;
if (p == NULL) { int32_t code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, &p, NULL);
return terrno; if (p == NULL || code != TSDB_CODE_SUCCESS) {
return code;
} }
ASSERT(p == pBlock); ASSERT(p == pBlock);
@ -1351,7 +1352,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
} }
if (hasNext) { if (hasNext) {
/*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL); SSDataBlock* p = NULL;
code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, &p, NULL);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
} }
@ -2477,27 +2483,54 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl
} }
} }
int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff, int32_t* pLen) {
int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); int32_t code = TSDB_CODE_SUCCESS;
len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); int32_t lino = 0;
*pBuff = taosMemoryCalloc(1, len); int32_t len = 0;
void* buf = *pBuff; code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len);
encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo);
return len; len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup);
*pBuff = taosMemoryCalloc(1, len);
if (!(*pBuff)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
void* buf = *pBuff;
encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup);
int32_t tmp = 0;
code = pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo, &tmp);
TSDB_CHECK_CODE(code, lino, _end);
*pLen = len;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) { void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (!pInfo->pState) { if (!pInfo->pState) {
return; return;
} }
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
void* pBuf = NULL; void* pBuf = NULL;
int32_t len = streamScanOperatorEncode(pInfo, &pBuf); int32_t len = 0;
code = streamScanOperatorEncode(pInfo, &pBuf, &len);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
saveStreamOperatorStateComplete(&pInfo->basic); saveStreamOperatorStateComplete(&pInfo->basic);
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
// other properties are recovered from the execution plan // other properties are recovered from the execution plan
@ -2929,8 +2962,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
} }
SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL); SSDataBlock* pBlock = NULL;
if (pBlock == NULL) { code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, &pBlock, NULL);
if (pBlock == NULL || code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, terrno); T_LONG_JMP(pTaskInfo->env, terrno);
} }
@ -3094,17 +3128,35 @@ static void destroyStreamScanOperatorInfo(void* param) {
} }
void streamScanReleaseState(SOperatorInfo* pOperator) { void streamScanReleaseState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamScanInfo* pInfo = pOperator->info; SStreamScanInfo* pInfo = pOperator->info;
void* pBuff = NULL;
if (!pInfo->pState) { if (!pInfo->pState) {
return; return;
} }
if (!pInfo->pUpdateInfo) { if (!pInfo->pUpdateInfo) {
return; return;
} }
int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); int32_t len = 0;
void* pBuff = taosMemoryCalloc(1, len); code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len);
pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo); TSDB_CHECK_CODE(code, lino, _end);
pBuff = taosMemoryCalloc(1, len);
if (!pBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
int32_t tmp = 0;
code = pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo, &tmp);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosMemoryFree(pBuff); taosMemoryFree(pBuff);
} }

View File

@ -68,9 +68,7 @@ void destroyStreamCountAggOperatorInfo(void* param) {
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; }
return pAggSup->windowCount != pAggSup->windowSliding;
}
void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
SBuffInfo* pBuffInfo) { SBuffInfo* pBuffInfo) {
@ -89,13 +87,14 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
ASSERT(pBuffInfo->pCur); ASSERT(pBuffInfo->pCur);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) { if (code == TSDB_CODE_FAILED) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
} }
} else { } else {
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount); pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) { if (code == TSDB_CODE_FAILED) {
@ -107,15 +106,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
pBuffInfo->rebuildWindow = true; pBuffInfo->rebuildWindow = true;
} }
} else { } else {
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist( code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size);
} }
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pCurWin->winInfo.isOutput = true; pCurWin->winInfo.isOutput = true;
} }
pCurWin->pWindowCount= pCurWin->pWindowCount =
(COUNT_TYPE*) ((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); (COUNT_TYPE*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE)));
if (*pCurWin->pWindowCount == pAggSup->windowCount) { if (*pCurWin->pWindowCount == pAggSup->windowCount) {
pBuffInfo->rebuildWindow = true; pBuffInfo->rebuildWindow = true;
@ -133,7 +133,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI
int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool* pRebuild) { SSHashObj* pStDeleted, bool* pRebuild) {
SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; SSessionKey sWinKey = pWinInfo->winInfo.sessionWin;
int32_t num = 0; int32_t num = 0;
for (int32_t i = start; i < rows; i++) { for (int32_t i = start; i < rows; i++) {
if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) { if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) {
num++; num++;
@ -181,7 +181,7 @@ void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, ESt
pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey); pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey);
} }
SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN}; SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
return; return;
@ -239,7 +239,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2);
SStreamStateCur* pCur = NULL; SStreamStateCur* pCur = NULL;
COUNT_TYPE slidingRows = 0; COUNT_TYPE slidingRows = 0;
for (int32_t i = 0; i < rows;) { for (int32_t i = 0; i < rows;) {
if (pInfo->ignoreExpiredData && if (pInfo->ignoreExpiredData &&
@ -269,7 +269,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
} }
getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range); getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range);
range.win.skey = TMIN(startTsCols[i], range.win.skey); range.win.skey = TMIN(startTsCols[i], range.win.skey);
range.win.ekey = TMAX(startTsCols[rows-1], range.win.ekey); range.win.ekey = TMAX(startTsCols[rows - 1], range.win.ekey);
uint64_t uid = 0; uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL); appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL);
break; break;
@ -289,8 +289,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
code = saveResult(curWin.winInfo, pStUpdated); code = saveResult(curWin.winInfo, pStUpdated);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
tstrerror(code));
break; break;
} }
} }
@ -397,13 +396,14 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
int32_t mapSize = 0; int32_t mapSize = 0;
buf = taosDecodeFixedI32(buf, &mapSize); buf = taosDecodeFixedI32(buf, &mapSize);
for (int32_t i = 0; i < mapSize; i++) { for (int32_t i = 0; i < mapSize; i++) {
SSessionKey key = {0}; SSessionKey key = {0};
SCountWindowInfo curWin = {0}; SCountWindowInfo curWin = {0};
buf = decodeSSessionKey(buf, &key); buf = decodeSSessionKey(buf, &key);
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo,
sizeof(SResultWindowInfo));
} }
// 2.twAggSup // 2.twAggSup
@ -441,9 +441,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; uint64_t* gpDatas = (uint64_t*)pGroupCol->pData;
SRowBuffPos* pPos = NULL; SRowBuffPos* pPos = NULL;
int32_t size = 0; int32_t size = 0;
for (int32_t i = 0; i < pBlock->info.rows; i++) { for (int32_t i = 0; i < pBlock->info.rows; i++) {
SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]};
SStreamStateCur* pCur = NULL; SStreamStateCur* pCur = NULL;
if (isSlidingCountWindow(pAggSup)) { if (isSlidingCountWindow(pAggSup)) {
pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount); pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount);
@ -452,7 +452,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
} }
while (1) { while (1) {
SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN}; SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size); int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void**)&pPos, &size);
if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) { if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) {
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
break; break;
@ -482,7 +482,7 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr
SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]}; SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]};
while (1) { while (1) {
SSessionKey curWin = {0}; SSessionKey curWin = {0};
int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin);
if (code == TSDB_CODE_FAILED) { if (code == TSDB_CODE_FAILED) {
break; break;
} }
@ -511,11 +511,11 @@ void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHa
} }
static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SStreamCountAggOperatorInfo* pInfo = pOperator->info; SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo; SOptrBasicInfo* pBInfo = &pInfo->binfo;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
@ -635,7 +635,7 @@ void streamCountReloadState(SOperatorInfo* pOperator) {
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME,
strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size);
TSKEY ts = *(TSKEY*)pBuf; TSKEY ts = *(TSKEY*)pBuf;
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
@ -650,11 +650,13 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) {
SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode; SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode;
int32_t numOfCols = 0; int32_t numOfCols = 0;
int32_t code = TSDB_CODE_OUT_OF_MEMORY; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo)); SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
goto _error; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
} }
pOperator->pTaskInfo = pTaskInfo; pOperator->pTaskInfo = pTaskInfo;
@ -664,18 +666,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
} }
SExprSupp* pExpSup = &pOperator->exprSupp; SExprSupp* pExpSup = &pOperator->exprSupp;
SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols); SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
pInfo->twAggSup = (STimeWindowAggSupp){ pInfo->twAggSup = (STimeWindowAggSupp){
.waterMark = pCountNode->window.watermark, .waterMark = pCountNode->window.watermark,
@ -686,12 +684,11 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
}; };
pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId; pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId;
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState,
pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup,
&pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
pInfo->streamAggSup.windowCount = pCountNode->windowCount; pInfo->streamAggSup.windowCount = pCountNode->windowCount;
pInfo->streamAggSup.windowSliding = pCountNode->windowSliding; pInfo->streamAggSup.windowSliding = pCountNode->windowSliding;
@ -709,7 +706,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->dataVersion = 0; pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) { if (!pInfo->historyWins) {
goto _error; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
} }
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
@ -735,8 +733,12 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
if (downstream) { if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
} }
return pOperator; return pOperator;
@ -747,6 +749,6 @@ _error:
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }

View File

@ -28,9 +28,9 @@
#include "tlog.h" #include "tlog.h"
#include "ttime.h" #include "ttime.h"
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) #define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" #define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" #define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
typedef struct SEventWinfowFlag { typedef struct SEventWinfowFlag {
bool startFlag; bool startFlag;
@ -86,10 +86,11 @@ void destroyStreamEventOperatorInfo(void* param) {
void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) { void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) {
char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize); char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize);
pWinInfo->pWinFlag = (SEventWinfowFlag*) pFlagInfo; pWinInfo->pWinFlag = (SEventWinfowFlag*)pFlagInfo;
} }
void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, SEventWindowInfo* pWinInfo) { void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos,
SEventWindowInfo* pWinInfo) {
pWinInfo->winInfo.sessionWin = *pKey; pWinInfo->winInfo.sessionWin = *pKey;
pWinInfo->winInfo.pStatePos = pPos; pWinInfo->winInfo.pStatePos = pPos;
setEventWindowFlag(pAggSup, pWinInfo); setEventWindowFlag(pAggSup, pWinInfo);
@ -122,22 +123,23 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.skey = ts;
pCurWin->winInfo.sessionWin.win.ekey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts;
SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); SStreamStateCur* pCur =
SSessionKey leftWinKey = {.groupId = groupId}; pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin);
void* pVal = NULL; SSessionKey leftWinKey = {.groupId = groupId};
int32_t len = 0; void* pVal = NULL;
int32_t len = 0;
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win) ) { if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) {
bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0);
setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin);
if(inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag) ) { if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) {
pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin); pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin);
goto _end; goto _end;
} }
} }
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
SSessionKey rightWinKey = {.groupId = groupId}; SSessionKey rightWinKey = {.groupId = groupId};
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len);
bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) {
@ -149,7 +151,7 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
} }
} }
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
pCurWin->pWinFlag->startFlag = start; pCurWin->pWinFlag->startFlag = start;
@ -164,7 +166,7 @@ _end:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pNextWinKey); SET_SESSION_WIN_KEY_INVALID(pNextWinKey);
} }
if(pCurWin->winInfo.pStatePos->needFree) { if (pCurWin->winInfo.pStatePos->needFree) {
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);
} }
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
@ -173,14 +175,14 @@ _end:
} }
int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey,
TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows, TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start,
SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) {
*pRebuild = false; *pRebuild = false;
if (!pWinInfo->pWinFlag->startFlag && !(starts[start]) ) { if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) {
return 1; return 1;
} }
TSKEY maxTs = INT64_MAX; TSKEY maxTs = INT64_MAX;
STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win; STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win;
if (pWinInfo->pWinFlag->endFlag) { if (pWinInfo->pWinFlag->endFlag) {
maxTs = pWin->ekey + 1; maxTs = pWin->ekey + 1;
@ -219,7 +221,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));
if (ends[i]) { if (ends[i]) {
if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey ) { if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) {
*pRebuild = true; *pRebuild = true;
} }
return i + 1 - start; return i + 1 - start;
@ -230,26 +232,28 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool addGap) { SSHashObj* pStDeleted, bool addGap) {
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
int32_t winNum = 0; int32_t winNum = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SResultRow* pCurResult = NULL; SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
while (1) { while (1) {
if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) { if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) {
break; break;
} }
SEventWindowInfo nextWinInfo = {0}; SEventWindowInfo nextWinInfo = {0};
getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo); getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo);
if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) ||
!inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) {
releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);
break; break;
} }
setEventWindowFlag(pAggSup, &nextWinInfo); setEventWindowFlag(pAggSup, &nextWinInfo);
compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, pStDeleted, false); compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated,
pStDeleted, false);
pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag;
winNum++; winNum++;
} }
@ -275,7 +279,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t winRows = 0; int32_t winRows = 0;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SColumnInfoData* pColStart = NULL; SColumnInfoData* pColStart = NULL;
SColumnInfoData* pColEnd = NULL; SColumnInfoData* pColEnd = NULL;
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
@ -290,7 +294,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
return; return;
} }
SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pStartCondInfo, &paramStart); code = filterSetDataFromSlotId(pInfo->pStartCondInfo, &paramStart);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("set data from start slotId error."); qError("set data from start slotId error.");
@ -299,7 +304,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t statusStart = 0; int32_t statusStart = 0;
filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart);
SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pEndCondInfo, &paramEnd); code = filterSetDataFromSlotId(pInfo->pEndCondInfo, &paramEnd);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("set data from end slotId error."); qError("set data from end slotId error.");
@ -320,20 +326,23 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t winIndex = 0; int32_t winIndex = 0;
bool allEqual = true; bool allEqual = true;
SEventWindowInfo curWin = {0}; SEventWindowInfo curWin = {0};
SSessionKey nextWinKey = {0}; SSessionKey nextWinKey = {0};
setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, &nextWinKey); setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin,
&nextWinKey);
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
bool rebuild = false; bool rebuild = false;
winRows = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i, winRows =
pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild);
ASSERT(winRows >= 1); ASSERT(winRows >= 1);
if (rebuild) { if (rebuild) {
uint64_t uid = 0; uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin);
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) { if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) &&
!isWindowIncomplete(&curWin)) {
saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
} }
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
@ -603,7 +612,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) {
char* pBuff = taosMemoryCalloc(1, resSize); char* pBuff = taosMemoryCalloc(1, resSize);
memcpy(pBuff, pInfo->historyWins->pData, winSize); memcpy(pBuff, pInfo->historyWins->pData, winSize);
memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
qDebug("===stream=== event window operator relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins)); qDebug("===stream=== event window operator relase state. save result count:%d",
(int32_t)taosArrayGetSize(pInfo->historyWins));
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME,
strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize); strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize);
pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);
@ -647,7 +657,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,
pSeKeyBuf[i].groupId, i); pSeKeyBuf[i].groupId, i);
getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo);
//event window has been deleted // event window has been deleted
if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) { if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) {
continue; continue;
} }
@ -659,7 +669,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId); curInfo.winInfo.sessionWin.groupId);
if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {
saveSessionOutputBuf(pAggSup, &curInfo.winInfo); saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
} }
@ -693,6 +703,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode; SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode;
int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId; int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo)); SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -783,20 +794,18 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0); code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0); code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
return pOperator; return pOperator;
@ -804,5 +813,6 @@ _error:
destroyStreamEventOperatorInfo(pInfo); destroyStreamEventOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }

View File

@ -129,7 +129,8 @@ static void destroyStreamFillOperatorInfo(void* param) {
pInfo->pRes = blockDataDestroy(pInfo->pRes); pInfo->pRes = blockDataDestroy(pInfo->pRes);
pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock);
pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList); taosArrayDestroy(pInfo->matchInfo.pList);
pInfo->matchInfo.pList = NULL;
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
} }

View File

@ -28,14 +28,18 @@
#include "tlog.h" #include "tlog.h"
#include "ttime.h" #include "ttime.h"
#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) #define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) #define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
#define IS_NORMAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) #define IS_NORMAL_INTERVAL_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) #define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) #define IS_NORMAL_SESSION_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) #define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
#define DEAULT_DELETE_MARK INT64_MAX #define DEAULT_DELETE_MARK INT64_MAX
#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState" #define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState"
@ -263,14 +267,16 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
if (chIds) { if (chIds) {
int32_t childId = getChildIndex(pBlock); int32_t childId = getChildIndex(pBlock);
if (pInvalidWins) { if (pInvalidWins) {
qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId); qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts,
winRes.groupId, childId);
taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0);
} }
SArray* chArray = *(void**)chIds; SArray* chArray = *(void**)chIds;
int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);
if (index != -1) { if (index != -1) {
qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, winGpId, childId); qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey,
winGpId, childId);
getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
continue; continue;
} }
@ -390,7 +396,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
} }
void destroyFlusedPos(void* pRes) { void destroyFlusedPos(void* pRes) {
SRowBuffPos* pPos = (SRowBuffPos*) pRes; SRowBuffPos* pPos = (SRowBuffPos*)pRes;
if (!pPos->needFree && !pPos->pRowBuff) { if (!pPos->needFree && !pPos->pRowBuff) {
taosMemoryFreeClear(pPos->pKey); taosMemoryFreeClear(pPos->pKey);
taosMemoryFree(pPos); taosMemoryFree(pPos);
@ -406,7 +412,8 @@ void clearGroupResInfo(SGroupResInfo* pGroupResInfo) {
} }
pGroupResInfo->freeItem = false; pGroupResInfo->freeItem = false;
} }
pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); taosArrayDestroy(pGroupResInfo->pRows);
pGroupResInfo->pRows = NULL;
pGroupResInfo->index = 0; pGroupResInfo->index = 0;
} }
@ -487,8 +494,11 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt
pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.parentType = type;
pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;
if (!pScanInfo->pUpdateInfo) { if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate,
pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at %d since %s", __func__, __LINE__, tstrerror(code));
}
} }
pScanInfo->interval = pInfo->interval; pScanInfo->interval = pInfo->interval;
@ -623,17 +633,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
} }
static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins, static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval,
int32_t numOfCh, SOperatorInfo* pOperator) { SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info; SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* tsData = (TSKEY*)pStartCol->pData; TSKEY* tsData = (TSKEY*)pStartCol->pData;
SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
TSKEY* tsEndData = (TSKEY*)pEndCol->pData; TSKEY* tsEndData = (TSKEY*)pEndCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
int32_t chId = getChildIndex(pBlock); int32_t chId = getChildIndex(pBlock);
bool res = false; bool res = false;
for (int32_t i = 0; i < pBlock->info.rows; i++) { for (int32_t i = 0; i < pBlock->info.rows; i++) {
TSKEY winTs = tsData[i]; TSKEY winTs = tsData[i];
while (winTs <= tsEndData[i]) { while (winTs <= tsEndData[i]) {
@ -649,7 +659,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
// pull data is over // pull data is over
taosArrayDestroy(chArray); taosArrayDestroy(chArray);
taosHashRemove(pMap, &winRes, sizeof(SWinKey)); taosHashRemove(pMap, &winRes, sizeof(SWinKey));
res =true; res = true;
qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts); qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts);
void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey));
@ -662,7 +672,8 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
.calWin.skey = nextWin.skey, .calWin.skey = nextWin.skey,
.calWin.ekey = nextWin.skey}; .calWin.ekey = nextWin.skey};
// add pull data request // add pull data request
qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", winRes.ts, winRes.groupId, numOfCh); qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d",
winRes.ts, winRes.groupId, numOfCh);
if (IS_MID_INTERVAL_OP(pOperator)) { if (IS_MID_INTERVAL_OP(pOperator)) {
SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;
taosArrayPush(pInfo->pMidPullDatas, &winRes); taosArrayPush(pInfo->pMidPullDatas, &winRes);
@ -670,7 +681,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
taosArrayPush(pInfo->pDelWins, &winRes); taosArrayPush(pInfo->pDelWins, &winRes);
addPullWindow(pMap, &winRes, numOfCh); addPullWindow(pMap, &winRes, numOfCh);
if (pInfo->destHasPrimaryKey) { if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);
} }
qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh);
} }
@ -689,7 +700,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
SWinKey* winKey = taosArrayGet(wins, i); SWinKey* winKey = taosArrayGet(wins, i);
STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
if (!chIds) { if (!chIds) {
SPullWindowInfo pull = { SPullWindowInfo pull = {
.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
@ -697,7 +708,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild);
if (pInfo->destHasPrimaryKey) { if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,winKey, sizeof(SWinKey), NULL, 0); tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0);
} }
qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild);
} }
@ -750,7 +761,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat
if (pBlock->info.id.groupId == 0) { if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = groupId; pBlock->info.id.groupId = groupId;
void* tbname = NULL; void* tbname = NULL;
if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, false) < 0) { if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,
false) < 0) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else { } else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -828,9 +840,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN
return startPos; return startPos;
} }
bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; }
return pInfo->primaryPkIndex != -1;
}
static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId,
SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) {
@ -851,23 +861,22 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData; tsCols = (int64_t*)pColDataInfo->pData;
void* pPkVal = NULL; void* pPkVal = NULL;
int32_t pkLen = 0; int32_t pkLen = 0;
SColumnInfoData* pPkColDataInfo = NULL; SColumnInfoData* pPkColDataInfo = NULL;
if (hasSrcPrimaryKeyCol(&pInfo->basic)) { if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
} }
if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) {
qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64,
",maxKey %" PRId64, pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex);
// timestamp of the data is incorrect // timestamp of the data is incorrect
if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {
qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
} }
} }
@ -915,7 +924,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild);
if (pInfo->destHasPrimaryKey) { if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);
} }
} }
} else { } else {
@ -961,7 +970,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
}; };
if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) {
tSimpleHashPut(pDeletedMap,&key, sizeof(SWinKey), NULL, 0); tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0);
} }
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
@ -1364,7 +1373,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
pInfo->binfo.pRes->info.type = pBlock->info.type; pInfo->binfo.pRes->info.type = pBlock->info.type;
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) { pBlock->info.type == STREAM_CLEAR) {
SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL;
doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap);
if (IS_FINAL_INTERVAL_OP(pOperator)) { if (IS_FINAL_INTERVAL_OP(pOperator)) {
@ -1398,7 +1407,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);
continue; continue;
} else if (pBlock->info.type == STREAM_RETRIEVE) { } else if (pBlock->info.type == STREAM_RETRIEVE) {
if(!IS_FINAL_INTERVAL_OP(pOperator)) { if (!IS_FINAL_INTERVAL_OP(pOperator)) {
pInfo->recvRetrive = true; pInfo->recvRetrive = true;
copyDataBlock(pInfo->pMidRetriveRes, pBlock); copyDataBlock(pInfo->pMidRetriveRes, pBlock);
pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;
@ -1524,8 +1533,8 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info; SStreamIntervalOperatorInfo* pInfo = pOperator->info;
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL && if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&
pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,
strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size);
if (code == 0) { if (code == 0) {
@ -1599,7 +1608,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pInfo->pState, &pTaskInfo->storageAPI.functionStore); pInfo->pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
@ -1754,27 +1763,37 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
} }
} }
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
SStreamPartitionOperatorInfo* pScanInfo = downstream->info; SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
pScanInfo->tsColIndex = tsColIndex; pScanInfo->tsColIndex = tsColIndex;
} }
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic);
return; return code;
} }
SStreamScanInfo* pScanInfo = downstream->info; SStreamScanInfo* pScanInfo = downstream->info;
pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
pScanInfo->pState = pAggSup->pState; pScanInfo->pState = pAggSup->pState;
if (!pScanInfo->pUpdateInfo) { if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,
pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen,
&pScanInfo->pUpdateInfo);
TSDB_CHECK_CODE(code, lino, _end);
} }
pScanInfo->twAggSup = *pTwSup; pScanInfo->twAggSup = *pTwSup;
pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;
pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex; pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static TSKEY sesionTs(void* pKey) { static TSKEY sesionTs(void* pKey) {
@ -2147,8 +2166,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;
void* pPkVal = NULL; void* pPkVal = NULL;
int32_t pkLen = 0; int32_t pkLen = 0;
SColumnInfoData* pPkColDataInfo = NULL; SColumnInfoData* pPkColDataInfo = NULL;
if (hasSrcPrimaryKeyCol(&pInfo->basic)) { if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
@ -2453,8 +2472,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa
pBlock->info.id.groupId = pKey->groupId; pBlock->info.id.groupId = pKey->groupId;
void* tbname = NULL; void* tbname = NULL;
if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,
&tbname, false) < 0) { false) < 0) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else { } else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -2934,7 +2953,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) {
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY));
@ -2967,7 +2986,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) {
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY));
@ -3017,6 +3036,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
int32_t numOfCols = 0; int32_t numOfCols = 0;
int32_t code = TSDB_CODE_OUT_OF_MEMORY; int32_t code = TSDB_CODE_OUT_OF_MEMORY;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -3111,8 +3131,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
if (downstream) { if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
} }
return pOperator; return pOperator;
@ -3123,6 +3147,7 @@ _error:
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }
@ -3573,7 +3598,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (!allEqual) { if (!allEqual) {
uint64_t uid = 0; uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
@ -3680,9 +3705,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
SSessionKey key = {0}; SSessionKey key = {0};
SResultWindowInfo winfo = {0}; SResultWindowInfo winfo = {0};
buf = decodeSSessionKey(buf, &key); buf = decodeSSessionKey(buf, &key);
pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize,
pAggSup->stateKeySize, compareStateKey, compareStateKey, (void**)&winfo.pStatePos,
(void**)&winfo.pStatePos, &pAggSup->resultRowSize); &pAggSup->resultRowSize);
buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
} }
@ -3876,7 +3901,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) {
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME,
strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
qDebug("===stream=== reload state. get result count:%d", num); qDebug("===stream=== reload state. get result count:%d", num);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
@ -3940,12 +3965,13 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey); SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _error; TSDB_CHECK_CODE(code, lino, _error);
} }
pInfo->stateCol = extractColumnFromColumnNode(pColNode); pInfo->stateCol = extractColumnFromColumnNode(pColNode);
@ -3954,9 +3980,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
} }
pInfo->twAggSup = (STimeWindowAggSupp){ pInfo->twAggSup = (STimeWindowAggSupp){
@ -3983,9 +4007,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize,
type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);
pInfo->pDelIterator = NULL; pInfo->pDelIterator = NULL;
@ -3998,7 +4021,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->dataVersion = 0; pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) { if (!pInfo->historyWins) {
goto _error; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
} }
if (pHandle) { if (pHandle) {
pInfo->isHistoryOp = pHandle->fillHistory; pInfo->isHistoryOp = pHandle->fillHistory;
@ -4025,17 +4049,20 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
return pOperator; return pOperator;
_error: _error:
destroyStreamStateOperatorInfo(pInfo); destroyStreamStateOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }
@ -4202,11 +4229,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
.precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
}; };
pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, pInfo->twAggSup =
.calTrigger = pIntervalPhyNode->window.triggerType, (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,
.maxTs = INT64_MIN, .calTrigger = pIntervalPhyNode->window.triggerType,
.minTs = INT64_MAX, .maxTs = INT64_MIN,
.deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; .minTs = INT64_MAX,
.deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};
ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
@ -4334,7 +4362,7 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
.ts = nextWin.skey, .ts = nextWin.skey,
.groupId = groupId, .groupId = groupId,
}; };
void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));
int32_t index = -1; int32_t index = -1;
SArray* chArray = NULL; SArray* chArray = NULL;
int32_t chId = 0; int32_t chId = 0;
@ -4406,8 +4434,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) {
int32_t size = taosArrayGetSize(wins); int32_t size = taosArrayGetSize(wins);
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
SWinKey* winKey = taosArrayGet(wins, i); SWinKey* winKey = taosArrayGet(wins, i);
void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));
if (!chIds) { if (!chIds) {
addPullWindow(pMidPullMap, winKey, numOfChild); addPullWindow(pMidPullMap, winKey, numOfChild);
qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild); qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild);
@ -4610,5 +4638,6 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
void setStreamOperatorCompleted(SOperatorInfo* pOperator) { void setStreamOperatorCompleted(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo),
getStreamOpName(pOperator->operatorType), pOperator->status);
} }

View File

@ -1136,9 +1136,9 @@ void destroyIntervalOperatorInfo(void* param) {
tdListFree(pInfo->binfo.resultRowInfo.openWindow); tdListFree(pInfo->binfo.resultRowInfo.openWindow);
pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); taosArrayDestroy(pInfo->pInterpCols);
pInfo->pInterpCols = NULL;
taosArrayDestroyEx(pInfo->pPrevValues, freeItem); taosArrayDestroyEx(pInfo->pPrevValues, freeItem);
pInfo->pPrevValues = NULL; pInfo->pPrevValues = NULL;
cleanupGroupResInfo(&pInfo->groupResInfo); cleanupGroupResInfo(&pInfo->groupResInfo);

View File

@ -1441,6 +1441,9 @@ _stddev_over:
} }
static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) { static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) {
if (IS_NULL_TYPE(pInput->type)) {
return;
}
pOutput->type = pInput->type; pOutput->type = pInput->type;
if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) { if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) {
pOutput->quadraticISum += pInput->quadraticISum; pOutput->quadraticISum += pInput->quadraticISum;

View File

@ -1061,6 +1061,13 @@ void nodesDestroyNode(SNode* pNode) {
pStmt->freeStbRowsCxtFunc(pStmt->pStbRowsCxt); pStmt->freeStbRowsCxtFunc(pStmt->pStbRowsCxt);
} }
taosMemoryFreeClear(pStmt->pStbRowsCxt); taosMemoryFreeClear(pStmt->pStbRowsCxt);
taosMemoryFreeClear(pStmt->pCreateTbInfo);
if (pStmt->destroyParseFileCxt) {
pStmt->destroyParseFileCxt(&pStmt->pParFileCxt);
}
taosCloseFile(&pStmt->fp); taosCloseFile(&pStmt->fp);
break; break;
} }
@ -1093,15 +1100,6 @@ void nodesDestroyNode(SNode* pNode) {
} }
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE: { case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE: {
SCreateSubTableFromFileClause* pStmt = (SCreateSubTableFromFileClause*)pNode; SCreateSubTableFromFileClause* pStmt = (SCreateSubTableFromFileClause*)pNode;
if (pStmt->aCreateTbData) {
taosArrayDestroy(pStmt->aCreateTbData);
}
if (pStmt->aTagIndexs) {
taosArrayDestroy(pStmt->aTagIndexs);
}
if (pStmt->fp) {
taosCloseFile(&pStmt->fp);
}
nodesDestroyList(pStmt->pSpecificTags); nodesDestroyList(pStmt->pSpecificTags);
break; break;
} }
@ -2205,6 +2203,17 @@ bool nodesIsRegularOp(const SOperatorNode* pOp) {
return false; return false;
} }
bool nodesIsMatchRegularOp(const SOperatorNode* pOp) {
switch (pOp->opType) {
case OP_TYPE_MATCH:
case OP_TYPE_NMATCH:
return true;
default:
break;
}
return false;
}
bool nodesIsBitwiseOp(const SOperatorNode* pOp) { bool nodesIsBitwiseOp(const SOperatorNode* pOp) {
switch (pOp->opType) { switch (pOp->opType) {
case OP_TYPE_BIT_AND: case OP_TYPE_BIT_AND:

View File

@ -29,6 +29,7 @@ extern "C" {
#define QUERY_SMA_OPTIMIZE_ENABLE 1 #define QUERY_SMA_OPTIMIZE_ENABLE 1
int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData); int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData);
int32_t continueCreateTbFromFile(SParseContext* pCxt, SQuery** pQuery);
int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); int32_t parse(SParseContext* pParseCxt, SQuery** pQuery);
int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache);
int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache);

View File

@ -353,6 +353,8 @@ end_opt(A) ::= END WITH TIMESTAMP NK_STRING(B).
cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B) cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); }
cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A); } cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A); }
cmd ::= CREATE TABLE not_exists_opt(B) USING full_table_name(C)
NK_LP tag_list_opt(D) NK_RP FILE NK_STRING(E). { pCxt->pRootNode = createCreateSubTableFromFileClause(pCxt, B, C, D, &E); }
cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B) cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B)
NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); } NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); }
cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); } cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); }
@ -385,15 +387,11 @@ alter_table_clause(A) ::=
%destructor multi_create_clause { nodesDestroyList($$); } %destructor multi_create_clause { nodesDestroyList($$); }
multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); } multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); }
multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). { A = addNodeToList(pCxt, B, C); } multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). { A = addNodeToList(pCxt, B, C); }
multi_create_clause(A) ::= create_from_file_clause(B). { A = createNodeList(pCxt, B); }
create_subtable_clause(A) ::= create_subtable_clause(A) ::=
not_exists_opt(B) full_table_name(C) USING full_table_name(D) not_exists_opt(B) full_table_name(C) USING full_table_name(D)
specific_cols_opt(E) TAGS NK_LP tags_literal_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); } specific_cols_opt(E) TAGS NK_LP tags_literal_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); }
create_from_file_clause(A) ::= not_exists_opt(B) USING full_table_name(C)
NK_LP tag_list_opt(D) NK_RP FILE NK_STRING(E). { A = createCreateSubTableFromFileClause(pCxt, B, C, D, &E); }
%type multi_drop_clause { SNodeList* } %type multi_drop_clause { SNodeList* }
%destructor multi_drop_clause { nodesDestroyList($$); } %destructor multi_drop_clause { nodesDestroyList($$); }
multi_drop_clause(A) ::= drop_table_clause(B). { A = createNodeList(pCxt, B); } multi_drop_clause(A) ::= drop_table_clause(B). { A = createNodeList(pCxt, B); }

View File

@ -310,6 +310,24 @@ static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCre
return code; return code;
} }
static int32_t collectMetaKeyFromCreateSubTableFromFile(SCollectMetaKeyCxt* pCxt,
SCreateSubTableFromFileClause* pClause) {
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code =
reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->useDbName, NULL,
AUTH_TYPE_WRITE, pCxt->pMetaCache);
}
return code;
}
static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableStmt* pStmt) { static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL; SNode* pNode = NULL;
@ -866,6 +884,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt); return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt);
case QUERY_NODE_CREATE_MULTI_TABLES_STMT: case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt); return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt);
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE:
return collectMetaKeyFromCreateSubTableFromFile(pCxt, (SCreateSubTableFromFileClause*)pStmt);
case QUERY_NODE_DROP_TABLE_STMT: case QUERY_NODE_DROP_TABLE_STMT:
return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt); return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt);
case QUERY_NODE_DROP_SUPER_TABLE_STMT: case QUERY_NODE_DROP_SUPER_TABLE_STMT:

View File

@ -112,7 +112,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
for (int i = 0; i < tags->numOfBound; ++i) { for (int i = 0; i < tags->numOfBound; ++i) {
SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSchema* pTagSchema = &pSchema[tags->pColIndex[i]];
SSmlKv* kv = taosArrayGet(cols, i); SSmlKv* kv = taosArrayGet(cols, i);
if (kv == NULL){
code = TSDB_CODE_SML_INVALID_DATA;
uError("SML smlBuildTagRow error kv is null");
goto end;
}
if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 ||
kv->type != pTagSchema->type) { kv->type != pTagSchema->type) {
code = TSDB_CODE_SML_INVALID_DATA; code = TSDB_CODE_SML_INVALID_DATA;
@ -120,7 +124,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
goto end; goto end;
} }
taosArrayPush(*tagName, pTagSchema->name); if (taosArrayPush(*tagName, pTagSchema->name) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML smlBuildTagRow error push tag name");
goto end;
}
STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
// strcpy(val.colName, pTagSchema->name); // strcpy(val.colName, pTagSchema->name);
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY ||
@ -141,7 +149,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
goto end; goto end;
} }
char buf[512] = {0}; char buf[512] = {0};
snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); (void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
taosMemoryFree(p); taosMemoryFree(p);
code = buildSyntaxErrMsg(msg, buf, kv->value); code = buildSyntaxErrMsg(msg, buf, kv->value);
goto end; goto end;
@ -149,9 +157,13 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
val.pData = p; val.pData = p;
val.nData = output; val.nData = output;
} else { } else {
memcpy(&val.i64, &(kv->value), kv->length); (void)memcpy(&val.i64, &(kv->value), kv->length);
}
if (taosArrayPush(pTagArray, &val) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML smlBuildTagRow error push tag array");
goto end;
} }
taosArrayPush(pTagArray, &val);
} }
code = tTagNew(pTagArray, 1, false, ppTag); code = tTagNew(pTagArray, 1, false, ppTag);
@ -166,20 +178,19 @@ end:
return code; return code;
} }
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) {
STableDataCxt* pTableCxt = NULL;
SVCreateTbReq* pCreateTbReq = NULL; SVCreateTbReq* pCreateTbReq = NULL;
int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return NULL; return ret;
} }
ret = initTableColSubmitData(pTableCxt); ret = initTableColSubmitData(*cxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return NULL; return ret;
} }
return pTableCxt; return TSDB_CODE_SUCCESS;
} }
void clearColValArraySml(SArray* pCols) { void clearColValArraySml(SArray* pCols) {
@ -197,6 +208,9 @@ void clearColValArraySml(SArray* pCols) {
int32_t smlBuildRow(STableDataCxt* pTableCxt) { int32_t smlBuildRow(STableDataCxt* pTableCxt) {
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
if (pRow == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS != ret) { if (TSDB_CODE_SUCCESS != ret) {
return ret; return ret;
@ -211,18 +225,22 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
int ret = TSDB_CODE_SUCCESS; int ret = TSDB_CODE_SUCCESS;
SSchema* pColSchema = schema + index; SSchema* pColSchema = schema + index;
SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
if (pVal == NULL) {
return TSDB_CODE_SUCCESS;
}
SSmlKv* kv = (SSmlKv*)data; SSmlKv* kv = (SSmlKv*)data;
if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 ||
kv->type != pColSchema->type) { kv->type != pColSchema->type) {
ret = TSDB_CODE_SML_INVALID_DATA; ret = TSDB_CODE_SML_INVALID_DATA;
char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1);
if (tmp) { if (tmp) {
memcpy(tmp, kv->key, kv->keyLen); (void)memcpy(tmp, kv->key, kv->keyLen);
uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name, uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name,
pColSchema->name, tDataTypes[pColSchema->type].name); pColSchema->name, tDataTypes[pColSchema->type].name);
taosMemoryFree(tmp); taosMemoryFree(tmp);
} else { } else {
uError("SML smlBuildCol out of memory"); uError("SML smlBuildCol out of memory");
ret = TSDB_CODE_OUT_OF_MEMORY;
} }
goto end; goto end;
} }
@ -256,9 +274,9 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length; pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length); pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
} else { } else {
memcpy(&pVal->value.val, &(kv->value), kv->length); (void)memcpy(&pVal->value.val, &(kv->value), kv->length);
} }
pVal->flag = CV_FLAG_VALUE; pVal->flag = CV_FLAG_VALUE;
@ -276,8 +294,13 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
SVCreateTbReq* pCreateTblReq = NULL; SVCreateTbReq* pCreateTblReq = NULL;
SArray* tagName = NULL; SArray* tagName = NULL;
insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); int ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags);
int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "init bound cols error");
goto end;
}
ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "bound tags error"); buildInvalidOperationMsg(&pBuf, "bound tags error");
goto end; goto end;
@ -299,7 +322,11 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
ttl); ttl);
pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1);
memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); if (pCreateTblReq->ctb.stbName == NULL){
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
(void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);
if (dataFormat) { if (dataFormat) {
STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj,
@ -345,11 +372,18 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
for (int32_t r = 0; r < rowNum; ++r) { for (int32_t r = 0; r < rowNum; ++r) {
void* rowData = taosArrayGetP(cols, r); void* rowData = taosArrayGetP(cols, r);
if (rowData == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
goto end;
}
// 1. set the parsed value from sql string // 1. set the parsed value from sql string
for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) {
SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]];
SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]);
if (pVal == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
goto end;
}
void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
if (p == NULL) { if (p == NULL) {
continue; continue;
@ -388,14 +422,22 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length; pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length); pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); if (NULL == pVal->value.pData) {
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
(void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
} else { } else {
memcpy(&pVal->value.val, &(kv->value), kv->length); (void)memcpy(&pVal->value.val, &(kv->value), kv->length);
} }
pVal->flag = CV_FLAG_VALUE; pVal->flag = CV_FLAG_VALUE;
} }
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
if (NULL == pRow) {
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS != ret) { if (TSDB_CODE_SUCCESS != ret) {
buildInvalidOperationMsg(&pBuf, "tRowBuild error"); buildInvalidOperationMsg(&pBuf, "tRowBuild error");
@ -415,11 +457,12 @@ end:
return ret; return ret;
} }
SQuery* smlInitHandle() { int32_t smlInitHandle(SQuery** query) {
*query = NULL;
SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == pQuery) { if (NULL == pQuery) {
uError("create pQuery error"); uError("create pQuery error");
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->haveResultSet = false; pQuery->haveResultSet = false;
@ -428,14 +471,22 @@ SQuery* smlInitHandle() {
if (NULL == stmt) { if (NULL == stmt) {
uError("create SVnodeModifyOpStmt error"); uError("create SVnodeModifyOpStmt error");
qDestroyQuery(pQuery); qDestroyQuery(pQuery);
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (stmt->pTableBlockHashObj == NULL){
uError("create pTableBlockHashObj error");
qDestroyQuery(pQuery);
nodesDestroyNode((SNode*)stmt);
return TSDB_CODE_OUT_OF_MEMORY;
}
stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeHashFunc = insDestroyTableDataCxtHashMap;
stmt->freeArrayFunc = insDestroyVgroupDataCxtList; stmt->freeArrayFunc = insDestroyVgroupDataCxtList;
pQuery->pRoot = (SNode*)stmt; pQuery->pRoot = (SNode*)stmt;
return pQuery; *query = pQuery;
return TSDB_CODE_SUCCESS;
} }
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) {

View File

@ -11889,7 +11889,8 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t createParOperatorNode(EOperatorType opType, const char* pLeftCol, const char* pRightCol, SNode** ppResOp) { static int32_t createParOperatorNode(EOperatorType opType, const char* pLeftCol, const char* pRightCol,
SNode** ppResOp) {
SOperatorNode* pOper = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); SOperatorNode* pOper = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR);
CHECK_POINTER_OUT_OF_MEM(pOper); CHECK_POINTER_OUT_OF_MEM(pOper);
@ -12694,7 +12695,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla
} }
static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFileClause* pStmt, static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFileClause* pStmt,
STableMeta* pSuperTableMeta) { STableMeta* pSuperTableMeta, SArray* aTagIndexs) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t numOfTags = getNumOfTags(pSuperTableMeta); int32_t numOfTags = getNumOfTags(pSuperTableMeta);
@ -12705,7 +12706,7 @@ static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFil
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
bool tbnameFound = false; bool tbnameFound = false;
SNode* pTagNode; SNode* pTagNode;
FOREACH(pTagNode, pStmt->pSpecificTags) { FOREACH(pTagNode, pStmt->pSpecificTags) {
@ -12757,14 +12758,14 @@ static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFil
goto _OUT; goto _OUT;
} }
if (NULL == taosArrayPush(pStmt->aTagIndexs, &idx)) { if (NULL == taosArrayPush(aTagIndexs, &idx)) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OUT; goto _OUT;
} }
} }
if (TSDB_CODE_SUCCESS == code && !tbnameFound) { if (TSDB_CODE_SUCCESS == code && !tbnameFound) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_ERROR); code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_ERROR);
} }
_OUT: _OUT:
@ -12772,21 +12773,19 @@ _OUT:
return code; return code;
} }
typedef struct { typedef struct SParseFileContext {
// refer SHashObj* pTbNameHash;
STableMeta* pSuperTableMeta; SArray* aTagNames;
SArray* pTagIndexs; bool tagNameFilled;
TdFilePtr fp; STableMeta* pStbMeta;
SArray* aTagIndexs;
// containers char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW];
SHashObj* pTbNameHash; SArray* aCreateTbData;
SArray* aTagNames;
bool tagNameFilled;
SArray* aTagVals;
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW];
// per line // per line
const char* pSql; const char* pSql;
SArray* aTagVals;
STag* pTag; STag* pTag;
SName ctbName; SName ctbName;
SVgroupInfo vg; SVgroupInfo vg;
@ -12810,48 +12809,48 @@ static int32_t fillVgroupInfo(SParseContext* pParseCxt, const SName* pName, SVgr
return code; return code;
} }
static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCtx) { static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int sz = taosArrayGetSize(pParFileCtx->pTagIndexs); int sz = taosArrayGetSize(pParFileCxt->aTagIndexs);
int32_t numOfTags = getNumOfTags(pParFileCtx->pSuperTableMeta); int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta);
uint8_t precision = getTableInfo(pParFileCtx->pSuperTableMeta).precision; uint8_t precision = getTableInfo(pParFileCxt->pStbMeta).precision;
SSchema* pSchemas = getTableTagSchema(pParFileCtx->pSuperTableMeta); SSchema* pSchemas = getTableTagSchema(pParFileCxt->pStbMeta);
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
const char* pSql = pParFileCtx->pSql; const char* pSql = pParFileCxt->pSql;
int32_t pos = 0; int32_t pos = 0;
SToken token = tStrGetToken(pSql, &pos, true, NULL); SToken token = tStrGetToken(pSql, &pos, true, NULL);
pParFileCtx->pSql += pos; pParFileCxt->pSql += pos;
if (TK_NK_RP == token.type) { if (TK_NK_RP == token.type) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
break; break;
} }
int16_t index = *(int16_t*)taosArrayGet(pParFileCtx->pTagIndexs, i); int16_t index = *(int16_t*)taosArrayGet(pParFileCxt->aTagIndexs, i);
if (index < numOfTags) { if (index < numOfTags) {
// parse tag // parse tag
const SSchema* pTagSchema = &pSchemas[index]; const SSchema* pTagSchema = &pSchemas[index];
code = checkAndTrimValue(&token, pParFileCtx->tmpTokenBuf, pMsgBuf, pTagSchema->type); code = checkAndTrimValue(&token, pParFileCxt->tmpTokenBuf, pMsgBuf, pTagSchema->type);
if (TSDB_CODE_SUCCESS == code && TK_NK_VARIABLE == token.type) { if (TSDB_CODE_SUCCESS == code && TK_NK_VARIABLE == token.type) {
code = buildInvalidOperationMsg(pMsgBuf, "not expected row value"); code = buildInvalidOperationMsg(pMsgBuf, "not expected row value");
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SArray* aTagNames = pParFileCtx->tagNameFilled ? NULL : pParFileCtx->aTagNames; SArray* aTagNames = pParFileCxt->tagNameFilled ? NULL : pParFileCxt->aTagNames;
code = parseTagValue(pMsgBuf, &pParFileCtx->pSql, precision, (SSchema*)pTagSchema, &token, code = parseTagValue(pMsgBuf, &pParFileCxt->pSql, precision, (SSchema*)pTagSchema, &token, aTagNames,
aTagNames, pParFileCtx->aTagVals, &pParFileCtx->pTag); pParFileCxt->aTagVals, &pParFileCxt->pTag);
} }
} else { } else {
// parse tbname // parse tbname
code = checkAndTrimValue(&token, pParFileCtx->tmpTokenBuf, pMsgBuf, TSDB_DATA_TYPE_BINARY); code = checkAndTrimValue(&token, pParFileCxt->tmpTokenBuf, pMsgBuf, TSDB_DATA_TYPE_BINARY);
if (TK_NK_VARIABLE == token.type) { if (TK_NK_VARIABLE == token.type) {
code = buildInvalidOperationMsg(pMsgBuf, "not expected tbname"); code = buildInvalidOperationMsg(pMsgBuf, "not expected tbname");
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
bool bFoundTbName = false; bool bFoundTbName = false;
code = parseTbnameToken(pMsgBuf, pParFileCtx->ctbName.tname, &token, &bFoundTbName); code = parseTbnameToken(pMsgBuf, pParFileCxt->ctbName.tname, &token, &bFoundTbName);
} }
} }
@ -12859,8 +12858,8 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCtx)
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pParFileCtx->tagNameFilled = true; pParFileCxt->tagNameFilled = true;
code = tTagNew(pParFileCtx->aTagVals, 1, false, &pParFileCtx->pTag); code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag);
} }
return code; return code;
@ -12873,171 +12872,262 @@ typedef struct {
SVgroupInfo vg; SVgroupInfo vg;
} SCreateTableData; } SCreateTableData;
static void clearTagValArrayFp(void *data) { static void clearTagValArrayFp(void* data) {
STagVal* p = (STagVal*)data; STagVal* p = (STagVal*)data;
if (IS_VAR_DATA_TYPE(p->type)) { if (IS_VAR_DATA_TYPE(p->type)) {
taosMemoryFreeClear(p->pData); taosMemoryFreeClear(p->pData);
} }
} }
static void clearCreateTbArrayFp(void *data) { static void clearCreateTbArrayFp(void* data) {
SCreateTableData* p = (SCreateTableData*)data; SCreateTableData* p = (SCreateTableData*)data;
taosMemoryFreeClear(p->pTag); taosMemoryFreeClear(p->pTag);
} }
static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFileContext* pParseFileCtx, static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFileContext* pParFileCxt, TdFilePtr fp,
SArray* aCreateTbData) { int32_t maxLineCount) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
char* pLine = NULL; char* pLine = NULL;
int64_t readLen = 0; int64_t readLen = 0;
while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pParseFileCtx->fp, &pLine)) != -1) { int32_t lineCount = 0;
while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(fp, &pLine)) != -1) {
if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) { if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) {
pLine[--readLen] = '\0'; pLine[--readLen] = '\0';
} }
if (readLen == 0) continue; if (readLen == 0) continue;
if (pLine[0] == '#') continue; if (pLine[0] == '#') continue; // ignore comment line begins with '#'
strtolower(pLine, pLine); strtolower(pLine, pLine);
pParseFileCtx->pSql = pLine; pParFileCxt->pSql = pLine;
code = parseOneStbRow(pMsgBuf, pParseFileCtx); code = parseOneStbRow(pMsgBuf, pParFileCxt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (taosHashGet(pParseFileCtx->pTbNameHash, pParseFileCtx->ctbName.tname, if (taosHashGet(pParFileCxt->pTbNameHash, pParFileCxt->ctbName.tname, strlen(pParFileCxt->ctbName.tname) + 1) !=
strlen(pParseFileCtx->ctbName.tname) + 1) != NULL) { NULL) {
taosMemoryFreeClear(pParseFileCtx->pTag); taosMemoryFreeClear(pParFileCxt->pTag);
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_DUPLICATED, pParseFileCtx->ctbName.tname); code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_DUPLICATED, pParFileCxt->ctbName.tname);
break; break;
} }
code = taosHashPut(pParseFileCtx->pTbNameHash, pParseFileCtx->ctbName.tname, code = taosHashPut(pParFileCxt->pTbNameHash, pParFileCxt->ctbName.tname, strlen(pParFileCxt->ctbName.tname) + 1,
strlen(pParseFileCtx->ctbName.tname) + 1, NULL, 0); NULL, 0);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = fillVgroupInfo(pParseCxt, &pParseFileCtx->ctbName, &pParseFileCtx->vg); code = fillVgroupInfo(pParseCxt, &pParFileCxt->ctbName, &pParFileCxt->vg);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SCreateTableData data = {.ctbName = pParseFileCtx->ctbName, SCreateTableData data = {.ctbName = pParFileCxt->ctbName,
.aTagNames = pParseFileCtx->aTagNames, .aTagNames = pParFileCxt->aTagNames,
.pTag = pParseFileCtx->pTag, .pTag = pParFileCxt->pTag,
.vg = pParseFileCtx->vg}; .vg = pParFileCxt->vg};
taosArrayPush(aCreateTbData, &data); taosArrayPush(pParFileCxt->aCreateTbData, &data);
} else { } else {
taosMemoryFreeClear(pParseFileCtx->pTag); taosMemoryFreeClear(pParFileCxt->pTag);
} }
pParseFileCtx->pTag = NULL; pParFileCxt->pTag = NULL;
taosArrayClearEx(pParseFileCtx->aTagVals, clearTagValArrayFp); taosArrayClearEx(pParFileCxt->aTagVals, clearTagValArrayFp);
lineCount++;
if (lineCount == maxLineCount) break;
} }
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
taosArrayClearEx(aCreateTbData, clearCreateTbArrayFp); taosArrayClearEx(pParFileCxt->aCreateTbData, clearCreateTbArrayFp);
} }
taosMemoryFree(pLine); taosMemoryFree(pLine);
return code; return code;
} }
static int32_t prepareReadFromFile(SCreateSubTableFromFileClause* pStmt) { static void destructParseFileContext(SParseFileContext** ppParFileCxt) {
if (NULL == ppParFileCxt || NULL == *ppParFileCxt) {
return;
}
SParseFileContext* pParFileCxt = *ppParFileCxt;
taosHashCleanup(pParFileCxt->pTbNameHash);
taosArrayDestroy(pParFileCxt->aTagNames);
taosMemoryFreeClear(pParFileCxt->pStbMeta);
taosArrayDestroy(pParFileCxt->aTagIndexs);
taosArrayDestroy(pParFileCxt->aCreateTbData);
taosArrayDestroy(pParFileCxt->aTagVals);
taosMemoryFree(pParFileCxt);
*ppParFileCxt = NULL;
return;
}
static int32_t constructParseFileContext(SCreateSubTableFromFileClause* pStmt, STableMeta* pSuperTableMeta,
int32_t acctId, SParseFileContext** ppParFileCxt) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pStmt->fp) {
pStmt->fp = taosOpenFile(pStmt->filePath, TD_FILE_READ | TD_FILE_STREAM); SParseFileContext* pParFileCxt = taosMemoryCalloc(1, sizeof(SParseFileContext));
if (NULL == pStmt->fp) { pParFileCxt->pStbMeta = pSuperTableMeta;
pParFileCxt->tagNameFilled = false;
pParFileCxt->pTag = NULL;
pParFileCxt->ctbName.type = TSDB_TABLE_NAME_T;
pParFileCxt->ctbName.acctId = acctId;
strcpy(pParFileCxt->ctbName.dbname, pStmt->useDbName);
if (NULL == pParFileCxt->pTbNameHash) {
pParFileCxt->pTbNameHash =
taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (!pParFileCxt->pTbNameHash) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
if (NULL == pParFileCxt->aTagNames) {
pParFileCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN);
if (NULL == pParFileCxt->aTagNames) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
if (NULL == pParFileCxt->aCreateTbData) {
pParFileCxt->aCreateTbData = taosArrayInit(16, sizeof(SCreateTableData));
if (NULL == pParFileCxt->aCreateTbData) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
if (NULL == pParFileCxt->aTagIndexs) {
pParFileCxt->aTagIndexs = taosArrayInit(pStmt->pSpecificTags->length, sizeof(int16_t));
if (!pParFileCxt->aTagIndexs) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
if (NULL == pParFileCxt->aTagVals) {
pParFileCxt->aTagVals = taosArrayInit(8, sizeof(STagVal));
if (!pParFileCxt->aTagVals) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
*ppParFileCxt = pParFileCxt;
return code;
_ERR:
destructParseFileContext(&pParFileCxt);
return code;
}
typedef struct SCreateTbInfo {
bool ignoreExists;
char useDbName[TSDB_DB_NAME_LEN];
char useTableName[TSDB_TABLE_NAME_LEN];
} SCreateTbInfo;
static int32_t prepareReadCsvFile(STranslateContext* pCxt, SCreateSubTableFromFileClause* pCreateStmt,
SVnodeModifyOpStmt* pModifyStmt) {
int32_t code = 0;
TdFilePtr fp = NULL;
SCreateTbInfo* pCreateInfo = NULL;
SParseFileContext* pParFileCxt = NULL;
if (NULL == pModifyStmt->fp) {
fp = taosOpenFile(pCreateStmt->filePath, TD_FILE_READ | TD_FILE_STREAM);
if (NULL == fp) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _ERR; goto _ERR;
} }
} }
if (NULL == pStmt->aCreateTbData) { {
pStmt->aCreateTbData = taosArrayInit(16, sizeof(SCreateTableData)); pCreateInfo = taosMemoryCalloc(1, sizeof(SCreateTbInfo));
if (NULL == pStmt->aCreateTbData) { if (NULL == pCreateInfo) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR; goto _ERR;
} }
pCreateInfo->ignoreExists = pCreateStmt->ignoreExists;
strncpy(pCreateInfo->useDbName, pCreateStmt->useDbName, TSDB_DB_NAME_LEN);
strncpy(pCreateInfo->useTableName, pCreateStmt->useTableName, TSDB_TABLE_NAME_LEN);
} }
if (NULL == pStmt->aTagIndexs) { {
pStmt->aTagIndexs = taosArrayInit(pStmt->pSpecificTags->length, sizeof(int16_t)); STableMeta* pSuperTableMeta = NULL;
if (!pStmt->aTagIndexs) { code = getTableMeta(pCxt, pCreateStmt->useDbName, pCreateStmt->useTableName, &pSuperTableMeta);
code = TSDB_CODE_OUT_OF_MEMORY; if (TSDB_CODE_SUCCESS != code) goto _ERR;
goto _ERR;
} code = constructParseFileContext(pCreateStmt, pSuperTableMeta, pCxt->pParseCxt->acctId, &pParFileCxt);
if (TSDB_CODE_SUCCESS != code) goto _ERR;
code = buildTagIndexForBindTags(&pCxt->msgBuf, pCreateStmt, pParFileCxt->pStbMeta, pParFileCxt->aTagIndexs);
if (TSDB_CODE_SUCCESS != code) goto _ERR;
} }
pModifyStmt->fp = fp;
pModifyStmt->fileProcessing = false;
pModifyStmt->pCreateTbInfo = pCreateInfo;
pModifyStmt->pParFileCxt = pParFileCxt;
return code; return code;
_ERR: _ERR:
taosCloseFile(&pStmt->fp); taosCloseFile(&fp);
taosArrayDestroy(pStmt->aCreateTbData); taosMemoryFreeClear(pCreateInfo);
taosArrayDestroy(pStmt->aTagIndexs); destructParseFileContext(&pParFileCxt);
return code; return code;
} }
static int32_t rewriteCreateSubTableFromFile(STranslateContext* pCxt, SCreateSubTableFromFileClause* pStmt, static int32_t resetParseFileContext(SParseFileContext* pParFileCxt) {
SHashObj* pVgroupHashmap) { taosArrayClear(pParFileCxt->aCreateTbData);
taosArrayClearEx(pParFileCxt->aTagVals, clearTagValArrayFp);
return TSDB_CODE_SUCCESS;
}
static int32_t createSubTableFromFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SVnodeModifyOpStmt* pModifyStmt) {
int32_t code = 0; int32_t code = 0;
STableMeta* pSuperTableMeta = NULL; SCreateTbInfo* pCreateInfo = pModifyStmt->pCreateTbInfo;
if (TSDB_CODE_SUCCESS == code) { SParseFileContext* pParFileCxt = pModifyStmt->pParFileCxt;
code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = prepareReadFromFile(pStmt); code = parseCsvFile(pMsgBuf, pParseCxt, pParFileCxt, pModifyStmt->fp, tsMaxInsertBatchRows);
} }
STableMeta* pSuperTableMeta = pParFileCxt->pStbMeta;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildTagIndexForBindTags(&pCxt->msgBuf, pStmt, pSuperTableMeta); int sz = taosArrayGetSize(pParFileCxt->aCreateTbData);
}
SParseFileContext parseFileCtx = {
.pSuperTableMeta = pSuperTableMeta, .fp = pStmt->fp, .pTagIndexs = pStmt->aTagIndexs};
parseFileCtx.pTbNameHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
parseFileCtx.aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN);
parseFileCtx.tagNameFilled = false;
parseFileCtx.aTagVals = taosArrayInit(8, sizeof(STagVal));
parseFileCtx.pTag = NULL;
parseFileCtx.ctbName.type = TSDB_TABLE_NAME_T;
parseFileCtx.ctbName.acctId = pCxt->pParseCxt->acctId;
strcpy(parseFileCtx.ctbName.dbname, pStmt->useDbName);
if (NULL == parseFileCtx.aTagNames || NULL == parseFileCtx.aTagVals || NULL == parseFileCtx.pTbNameHash) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OUT;
}
if (TSDB_CODE_SUCCESS == code) {
code = parseCsvFile(&pCxt->msgBuf, pCxt->pParseCxt, &parseFileCtx, pStmt->aCreateTbData);
}
if (TSDB_CODE_SUCCESS == code) {
int sz = taosArrayGetSize(pStmt->aCreateTbData);
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
SCreateTableData* pData = taosArrayGet(pStmt->aCreateTbData, i); SCreateTableData* pData = taosArrayGet(pParFileCxt->aCreateTbData, i);
code = collectUseTable(&pData->ctbName, pCxt->pTargetTables); // code = collectUseTable(&pData->ctbName, pCxt->pTargetTables);
if (TSDB_CODE_SUCCESS != code) { // if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(pData->pTag); // taosMemoryFree(pData->pTag);
} // }
code = addCreateTbReqIntoVgroup(pVgroupHashmap, pStmt->useDbName, pSuperTableMeta->uid, pStmt->useTableName, code = addCreateTbReqIntoVgroup(pModifyStmt->pVgroupsHashObj, pCreateInfo->useDbName, pSuperTableMeta->uid,
pData->ctbName.tname, pData->aTagNames, pSuperTableMeta->tableInfo.numOfTags, pCreateInfo->useTableName, pData->ctbName.tname, pData->aTagNames,
pData->pTag, TSDB_DEFAULT_TABLE_TTL, NULL, pStmt->ignoreExists, &pData->vg); pSuperTableMeta->tableInfo.numOfTags, pData->pTag, TSDB_DEFAULT_TABLE_TTL, NULL,
pCreateInfo->ignoreExists, &pData->vg);
}
if (TSDB_CODE_SUCCESS == code) {
pModifyStmt->fileProcessing = (sz == tsMaxInsertBatchRows);
} }
} }
_OUT: (void)resetParseFileContext(pModifyStmt->pParFileCxt);
taosMemoryFreeClear(pSuperTableMeta);
taosHashCleanup(parseFileCtx.pTbNameHash);
taosArrayDestroy(parseFileCtx.aTagNames);
taosArrayDestroy(parseFileCtx.aTagVals);
return code; return code;
} }
@ -13074,13 +13164,8 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode; SNode* pNode;
FOREACH(pNode, pStmt->pSubTables) { FOREACH(pNode, pStmt->pSubTables) {
if (pNode->type == QUERY_NODE_CREATE_SUBTABLE_CLAUSE) { SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode; code = rewriteCreateSubTable(pCxt, pClause, pVgroupHashmap);
code = rewriteCreateSubTable(pCxt, pClause, pVgroupHashmap);
} else {
SCreateSubTableFromFileClause* pClause = (SCreateSubTableFromFileClause*)pNode;
code = rewriteCreateSubTableFromFile(pCxt, pClause, pVgroupHashmap);
}
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
taosHashCleanup(pVgroupHashmap); taosHashCleanup(pVgroupHashmap);
return code; return code;
@ -13096,6 +13181,75 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
return rewriteToVnodeModifyOpStmt(pQuery, pBufArray); return rewriteToVnodeModifyOpStmt(pQuery, pBufArray);
} }
static int32_t rewriteCreateTableFromFile(STranslateContext* pCxt, SQuery* pQuery) {
SVnodeModifyOpStmt* pModifyStmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT);
if (pModifyStmt == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pModifyStmt->sqlNodeType = nodeType(pQuery->pRoot);
pModifyStmt->fileProcessing = false;
pModifyStmt->destroyParseFileCxt = destructParseFileContext;
pModifyStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (NULL == pModifyStmt->pVgroupsHashObj) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosHashSetFreeFp(pModifyStmt->pVgroupsHashObj, destroyCreateTbReqBatch);
SCreateSubTableFromFileClause* pCreateStmt = (SCreateSubTableFromFileClause*)pQuery->pRoot;
int32_t code = prepareReadCsvFile(pCxt, pCreateStmt, pModifyStmt);
if (TSDB_CODE_SUCCESS != code) {
taosHashCleanup(pModifyStmt->pVgroupsHashObj);
return code;
}
code = createSubTableFromFile(&pCxt->msgBuf, pCxt->pParseCxt, pModifyStmt);
if (TSDB_CODE_SUCCESS != code) {
taosHashCleanup(pModifyStmt->pVgroupsHashObj);
return code;
}
SArray* pBufArray = serializeVgroupsCreateTableBatch(pModifyStmt->pVgroupsHashObj);
taosHashClear(pModifyStmt->pVgroupsHashObj);
if (NULL == pBufArray) {
taosHashCleanup(pModifyStmt->pVgroupsHashObj);
return TSDB_CODE_OUT_OF_MEMORY;
}
pModifyStmt->pDataBlocks = pBufArray;
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = (SNode*)pModifyStmt;
return TSDB_CODE_SUCCESS;
}
int32_t continueCreateTbFromFile(SParseContext* pParseCxt, SQuery** pQuery) {
SVnodeModifyOpStmt* pModifyStmt = (SVnodeModifyOpStmt*)(*pQuery)->pRoot;
SMsgBuf tmpBuf = {0};
tmpBuf.buf = taosMemoryMalloc(1024);
int32_t code = createSubTableFromFile(&tmpBuf, pParseCxt, pModifyStmt);
if (TSDB_CODE_SUCCESS != code) goto _OUT;
SArray* pBufArray = serializeVgroupsCreateTableBatch(pModifyStmt->pVgroupsHashObj);
taosHashClear(pModifyStmt->pVgroupsHashObj);
if (NULL == pBufArray) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OUT;
}
pModifyStmt->pDataBlocks = pBufArray;
(*pQuery)->execStage = QUERY_EXEC_STAGE_SCHEDULE;
if (!pModifyStmt->fileProcessing) {
(*pQuery)->execMode = QUERY_EXEC_MODE_EMPTY_RESULT;
}
code = TSDB_CODE_SUCCESS;
_OUT:
taosMemoryFreeClear(tmpBuf.buf);
return code;
}
typedef struct SVgroupDropTableBatch { typedef struct SVgroupDropTableBatch {
SVDropTbBatchReq req; SVDropTbBatchReq req;
SVgroupInfo info; SVgroupInfo info;
@ -13798,7 +13952,8 @@ static int32_t createParWhenThenNode(SNode* pWhen, SNode* pThen, SNode** ppResWh
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t createParCaseWhenNode(SNode* pCase, SNodeList* pWhenThenList, SNode* pElse, const char* pAias, SNode** ppResCaseWhen) { static int32_t createParCaseWhenNode(SNode* pCase, SNodeList* pWhenThenList, SNode* pElse, const char* pAias,
SNode** ppResCaseWhen) {
SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)nodesMakeNode(QUERY_NODE_CASE_WHEN); SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)nodesMakeNode(QUERY_NODE_CASE_WHEN);
CHECK_POINTER_OUT_OF_MEM(pCaseWhen); CHECK_POINTER_OUT_OF_MEM(pCaseWhen);
@ -13813,7 +13968,8 @@ static int32_t createParCaseWhenNode(SNode* pCase, SNodeList* pWhenThenList, SNo
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t createParFunctionNode(const char* pFunName, const char* pAias, SNodeList* pParameterList, SNode** ppResFunc) { static int32_t createParFunctionNode(const char* pFunName, const char* pAias, SNodeList* pParameterList,
SNode** ppResFunc) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
CHECK_POINTER_OUT_OF_MEM(pFunc); CHECK_POINTER_OUT_OF_MEM(pFunc);
strcpy(pFunc->functionName, pFunName); strcpy(pFunc->functionName, pFunName);
@ -13854,13 +14010,13 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
} }
} }
SValueNode* pValNode = nodesMakeValueNodeFromString("leader"); SValueNode* pValNode = nodesMakeValueNodeFromString("leader");
CHECK_POINTER_OUT_OF_MEM(pValNode); CHECK_POINTER_OUT_OF_MEM(pValNode);
SNode* pCond1 = NULL; SNode* pCond1 = NULL;
SNode* pCond2 = NULL; SNode* pCond2 = NULL;
SNode* pCond3 = NULL; SNode* pCond3 = NULL;
SNode* pCond4 = NULL; SNode* pCond4 = NULL;
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "v1_status", (SNode*)pValNode, &pCond1)); CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "v1_status", (SNode*)pValNode, &pCond1));
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "v2_status", (SNode*)pValNode, &pCond2)); CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "v2_status", (SNode*)pValNode, &pCond2));
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "v3_status", (SNode*)pValNode, &pCond3)); CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "v3_status", (SNode*)pValNode, &pCond3));
@ -13885,16 +14041,16 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
SNode* pElse = nodesMakeValueNodeFromInt32(0); SNode* pElse = nodesMakeValueNodeFromInt32(0);
CHECK_POINTER_OUT_OF_MEM(pElse); CHECK_POINTER_OUT_OF_MEM(pElse);
// case when (v1_status = "leader" or v2_status = "lead er" or v3_status = "leader" or v4_status = "leader") then 1 else 0 end // case when (v1_status = "leader" or v2_status = "lead er" or v3_status = "leader" or v4_status = "leader") then 1
// else 0 end
SNode* pCaseWhen = NULL; SNode* pCaseWhen = NULL;
CHECK_RES_OUT_OF_MEM(createParCaseWhenNode(NULL, pWhenThenlist, pElse, NULL, &pCaseWhen)); CHECK_RES_OUT_OF_MEM(createParCaseWhenNode(NULL, pWhenThenlist, pElse, NULL, &pCaseWhen));
SNodeList* pParaList = NULL; SNodeList* pParaList = NULL;
CHECK_RES_OUT_OF_MEM(createParListNode(pCaseWhen, &pParaList)); CHECK_RES_OUT_OF_MEM(createParListNode(pCaseWhen, &pParaList));
// sum( case when ... end) as leader_col // sum( case when ... end) as leader_col
SNode* pSumFun = NULL; SNode* pSumFun = NULL;
const char* pSumColAlias = "leader_col"; const char* pSumColAlias = "leader_col";
CHECK_RES_OUT_OF_MEM(createParFunctionNode("sum", pSumColAlias, pParaList, &pSumFun)); CHECK_RES_OUT_OF_MEM(createParFunctionNode("sum", pSumColAlias, pParaList, &pSumFun));
@ -13904,7 +14060,7 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
CHECK_RES_OUT_OF_MEM(createParListNode(pPara1, &pParaList)); CHECK_RES_OUT_OF_MEM(createParListNode(pPara1, &pParaList));
// count(1) as count_col // count(1) as count_col
SNode* pCountFun = NULL; SNode* pCountFun = NULL;
const char* pCountColAlias = "count_col"; const char* pCountColAlias = "count_col";
CHECK_RES_OUT_OF_MEM(createParFunctionNode("count", pCountColAlias, pParaList, &pCountFun)); CHECK_RES_OUT_OF_MEM(createParFunctionNode("count", pCountColAlias, pParaList, &pCountFun));
@ -13914,11 +14070,13 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
SSelectStmt* pSubSelect = NULL; SSelectStmt* pSubSelect = NULL;
// select sum( case when .... end) as leader_col, count(*) as count_col from information_schema.ins_vgroups // select sum( case when .... end) as leader_col, count(*) as count_col from information_schema.ins_vgroups
CHECK_RES_OUT_OF_MEM(createSimpleSelectStmtFromProjList(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VGROUPS, pProjList, &pSubSelect)); CHECK_RES_OUT_OF_MEM(
createSimpleSelectStmtFromProjList(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VGROUPS, pProjList, &pSubSelect));
if (pDbName && pDbName[0] != 0) { if (pDbName && pDbName[0] != 0) {
// for show db.alive // for show db.alive
// select sum( case when .... end) as leader_col, count(*) as count_col from information_schema.ins_vgroups where db_name = "..." // select sum( case when .... end) as leader_col, count(*) as count_col from information_schema.ins_vgroups where
// db_name = "..."
SNode* pDbCond = NULL; SNode* pDbCond = NULL;
pValNode = nodesMakeValueNodeFromString(pDbName); pValNode = nodesMakeValueNodeFromString(pDbName);
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "db_name", (SNode*)pValNode, &pDbCond)); CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "db_name", (SNode*)pValNode, &pDbCond));
@ -13928,14 +14086,12 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
pSubSelect->pWhere = pDbCond; pSubSelect->pWhere = pDbCond;
} }
pCond1 = NULL; pCond1 = NULL;
CHECK_RES_OUT_OF_MEM(createParOperatorNode(OP_TYPE_EQUAL, pSumColAlias, pCountColAlias, &pCond1)); CHECK_RES_OUT_OF_MEM(createParOperatorNode(OP_TYPE_EQUAL, pSumColAlias, pCountColAlias, &pCond1));
pCond2 = NULL; pCond2 = NULL;
SNode* pTempVal = nodesMakeValueNodeFromInt32(0); SNode* pTempVal = nodesMakeValueNodeFromInt32(0);
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_GREATER_THAN, pSumColAlias, pTempVal, &pCond2)); CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_GREATER_THAN, pSumColAlias, pTempVal, &pCond2));
//leader_col = count_col and leader_col > 0 // leader_col = count_col and leader_col > 0
pTemp1 = NULL; pTemp1 = NULL;
CHECK_RES_OUT_OF_MEM(createLogicCondNode(pCond1, pCond2, &pTemp1, LOGIC_COND_TYPE_AND)); CHECK_RES_OUT_OF_MEM(createLogicCondNode(pCond1, pCond2, &pTemp1, LOGIC_COND_TYPE_AND));
@ -13961,7 +14117,8 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
CHECK_RES_OUT_OF_MEM(createParWhenThenNode(pTemp2, pThen, &pWhenThen)); CHECK_RES_OUT_OF_MEM(createParWhenThenNode(pTemp2, pThen, &pWhenThen));
CHECK_RES_OUT_OF_MEM(nodesListStrictAppend(pWhenThenlist, pWhenThen)); CHECK_RES_OUT_OF_MEM(nodesListStrictAppend(pWhenThenlist, pWhenThen));
// case when leader_col = count_col and count_col > 0 then 1 when leader_col < count_col and count_col > 0 then 2 else 0 end as status // case when leader_col = count_col and count_col > 0 then 1 when leader_col < count_col and count_col > 0 then 2 else
// 0 end as status
pCaseWhen = NULL; pCaseWhen = NULL;
pElse = nodesMakeValueNodeFromInt32(0); pElse = nodesMakeValueNodeFromInt32(0);
CHECK_POINTER_OUT_OF_MEM(pElse); CHECK_POINTER_OUT_OF_MEM(pElse);
@ -13973,7 +14130,6 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
SNode* pTempTblNode = NULL; SNode* pTempTblNode = NULL;
CHECK_RES_OUT_OF_MEM(createParTempTableNode(pSubSelect, &pTempTblNode)); CHECK_RES_OUT_OF_MEM(createParTempTableNode(pSubSelect, &pTempTblNode));
SSelectStmt* pStmt = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); SSelectStmt* pStmt = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT);
CHECK_POINTER_OUT_OF_MEM(pStmt); CHECK_POINTER_OUT_OF_MEM(pStmt);
pStmt->pProjectionList = pProjList; pStmt->pProjectionList = pProjList;
@ -14047,6 +14203,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_CREATE_MULTI_TABLES_STMT: case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
code = rewriteCreateMultiTable(pCxt, pQuery); code = rewriteCreateMultiTable(pCxt, pQuery);
break; break;
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE:
code = rewriteCreateTableFromFile(pCxt, pQuery);
break;
case QUERY_NODE_DROP_TABLE_STMT: case QUERY_NODE_DROP_TABLE_STMT:
code = rewriteDropTable(pCxt, pQuery); code = rewriteDropTable(pCxt, pQuery);
break; break;

View File

@ -223,7 +223,9 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Tag name:%s duplicated"; return "Tag name:%s duplicated";
case TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC: case TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC:
return "Some functions cannot appear in the select list at the same time"; return "Some functions cannot appear in the select list at the same time";
default: case TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR:
return "Syntax error in regular expression";
default:
return "Unknown error"; return "Unknown error";
} }
} }

View File

@ -48,6 +48,33 @@ bool qIsInsertValuesSql(const char* pStr, size_t length) {
return false; return false;
} }
bool qIsCreateTbFromFileSql(const char* pStr, size_t length) {
if (NULL == pStr) {
return false;
}
const char* pSql = pStr;
int32_t index = 0;
SToken t = tStrGetToken((char*)pStr, &index, false, NULL);
if (TK_CREATE != t.type) {
return false;
}
do {
pStr += index;
index = 0;
t = tStrGetToken((char*)pStr, &index, false, NULL);
if (TK_FILE == t.type) {
return true;
}
if (0 == t.type || 0 == t.n) {
break;
}
} while (pStr - pSql < length);
return false;
}
bool qParseDbName(const char* pStr, size_t length, char** pDbName) { bool qParseDbName(const char* pStr, size_t length, char** pDbName) {
(void) length; (void) length;
int32_t index = 0; int32_t index = 0;
@ -239,11 +266,19 @@ static int32_t parseQuerySyntax(SParseContext* pCxt, SQuery** pQuery, struct SCa
return code; return code;
} }
static int32_t parseCreateTbFromFileSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) {
if (NULL == *pQuery) return parseQuerySyntax(pCxt, pQuery, pCatalogReq);
return continueCreateTbFromFile(pCxt, pQuery);
}
int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) {
int32_t code = nodesAcquireAllocator(pCxt->allocatorId); int32_t code = nodesAcquireAllocator(pCxt->allocatorId);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) {
code = parseInsertSql(pCxt, pQuery, pCatalogReq, NULL); code = parseInsertSql(pCxt, pQuery, pCatalogReq, NULL);
} else if (qIsCreateTbFromFileSql(pCxt->pSql, pCxt->sqlLen)) {
code = parseCreateTbFromFileSyntax(pCxt, pQuery, pCatalogReq);
} else { } else {
code = parseQuerySyntax(pCxt, pQuery, pCatalogReq); code = parseQuerySyntax(pCxt, pQuery, pCatalogReq);
} }
@ -322,7 +357,7 @@ void destoryCatalogReq(SCatalogReq *pCatalogReq) {
taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq); taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq);
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
taosArrayDestroyEx(pCatalogReq->pView, destoryTablesReq); taosArrayDestroyEx(pCatalogReq->pView, destoryTablesReq);
#endif #endif
taosArrayDestroyEx(pCatalogReq->pTableTSMAs, destoryTablesReq); taosArrayDestroyEx(pCatalogReq->pTableTSMAs, destoryTablesReq);
taosArrayDestroyEx(pCatalogReq->pTSMAs, destoryTablesReq); taosArrayDestroyEx(pCatalogReq->pTSMAs, destoryTablesReq);
} }

File diff suppressed because it is too large Load Diff

View File

@ -539,7 +539,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
pJoin->pJLimit = nodesCloneNode(pJoinTable->pJLimit); pJoin->pJLimit = nodesCloneNode(pJoinTable->pJLimit);
pJoin->addPrimEqCond = nodesCloneNode(pJoinTable->addPrimCond); pJoin->addPrimEqCond = nodesCloneNode(pJoinTable->addPrimCond);
pJoin->node.pChildren = nodesMakeList(); pJoin->node.pChildren = nodesMakeList();
pJoin->seqWinGroup = (JOIN_STYPE_WIN == pJoinTable->subType) && (pSelect->hasAggFuncs || pSelect->hasIndefiniteRowsFunc); pJoin->seqWinGroup = (JOIN_STYPE_WIN == pJoinTable->subType) && (pSelect->hasAggFuncs || pSelect->hasIndefiniteRowsFunc);
if (NULL == pJoin->node.pChildren) { if (NULL == pJoin->node.pChildren) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -619,7 +619,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
} }
} }
#else #else
// set the output // set the output
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SNodeList* pColList = NULL; SNodeList* pColList = NULL;
@ -762,7 +762,7 @@ static int32_t addWinJoinPrimKeyToAggFuncs(SSelectStmt* pSelect, SNodeList** pLi
pCol->hasIndex = (pColSchema != NULL && IS_IDX_ON(pColSchema)); pCol->hasIndex = (pColSchema != NULL && IS_IDX_ON(pColSchema));
pCol->node.resType.type = pColSchema->type; pCol->node.resType.type = pColSchema->type;
pCol->node.resType.bytes = pColSchema->bytes; pCol->node.resType.bytes = pColSchema->bytes;
pCol->node.resType.precision = pProbeTable->pMeta->tableInfo.precision; pCol->node.resType.precision = pProbeTable->pMeta->tableInfo.precision;
SNode* pFunc = (SNode*)createGroupKeyAggFunc(pCol); SNode* pFunc = (SNode*)createGroupKeyAggFunc(pCol);
@ -841,7 +841,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0;
pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0;
pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList; pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pAgg; *pLogicNode = (SLogicNode*)pAgg;
} else { } else {
@ -1708,6 +1708,7 @@ static int32_t getMsgType(ENodeType sqlType) {
switch (sqlType) { switch (sqlType) {
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
case QUERY_NODE_CREATE_MULTI_TABLES_STMT: case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE:
return TDMT_VND_CREATE_TABLE; return TDMT_VND_CREATE_TABLE;
case QUERY_NODE_DROP_TABLE_STMT: case QUERY_NODE_DROP_TABLE_STMT:
return TDMT_VND_DROP_TABLE; return TDMT_VND_DROP_TABLE;

View File

@ -218,8 +218,8 @@ typedef struct SQWorkerMgmt {
#define QW_IDS() sId, qId, tId, rId, eId #define QW_IDS() sId, qId, tId, rId, eId
#define QW_FPARAMS() mgmt, QW_IDS() #define QW_FPARAMS() mgmt, QW_IDS()
#define QW_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n) #define QW_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n)
#define QW_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) #define QW_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n)
#define QW_STAT_GET(_item) atomic_load_64(&(_item)) #define QW_STAT_GET(_item) atomic_load_64(&(_item))
#define QW_GET_EVENT(ctx, event) atomic_load_8(&(ctx)->events[event]) #define QW_GET_EVENT(ctx, event) atomic_load_8(&(ctx)->events[event])
@ -250,7 +250,7 @@ typedef struct SQWorkerMgmt {
} while (0) } while (0)
#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) #define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code)
#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) #define QW_UPDATE_RSP_CODE(ctx, code) (void)atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) #define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY)
#define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch) #define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch)
@ -391,7 +391,7 @@ void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx);
int32_t qwKillTaskHandle(SQWTaskCtx *ctx, int32_t rspCode); int32_t qwKillTaskHandle(SQWTaskCtx *ctx, int32_t rspCode);
int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status, bool dynamicTask); int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status, bool dynamicTask);
int32_t qwDropTask(QW_FPARAMS_DEF); int32_t qwDropTask(QW_FPARAMS_DEF);
void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx); int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx);
int32_t qwOpenRef(void); int32_t qwOpenRef(void);
void qwSetHbParam(int64_t refId, SQWHbParam **pParam); void qwSetHbParam(int64_t refId, SQWHbParam **pParam);
int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type); int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type);

View File

@ -173,8 +173,21 @@ int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int
if (pEpSet) { if (pEpSet) {
contLen = tSerializeSEpSet(NULL, 0, pEpSet); contLen = tSerializeSEpSet(NULL, 0, pEpSet);
if (contLen < 0) {
qError("tSerializeSEpSet failed, code:%x", terrno);
return terrno;
}
rsp = rpcMallocCont(contLen); rsp = rpcMallocCont(contLen);
tSerializeSEpSet(rsp, contLen, pEpSet); if (NULL == rsp) {
qError("rpcMallocCont %d failed, code:%x", contLen, terrno);
return terrno;
}
contLen = tSerializeSEpSet(rsp, contLen, pEpSet);
if (contLen < 0) {
qError("tSerializeSEpSet second failed, code:%x", terrno);
return terrno;
}
} }
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
@ -208,28 +221,28 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) {
SEpSet epSet = {0}; SEpSet epSet = {0};
epSet.inUse = 1; epSet.inUse = 1;
epSet.numOfEps = 3; epSet.numOfEps = 3;
strcpy(epSet.eps[0].fqdn, "localhost"); TAOS_STRCPY(epSet.eps[0].fqdn, "localhost");
epSet.eps[0].port = 7100; epSet.eps[0].port = 7100;
strcpy(epSet.eps[1].fqdn, "localhost"); TAOS_STRCPY(epSet.eps[1].fqdn, "localhost");
epSet.eps[1].port = 7200; epSet.eps[1].port = 7200;
strcpy(epSet.eps[2].fqdn, "localhost"); TAOS_STRCPY(epSet.eps[2].fqdn, "localhost");
epSet.eps[2].port = 7300; epSet.eps[2].port = 7300;
ctx->phase = QW_PHASE_POST_QUERY; ctx->phase = QW_PHASE_POST_QUERY;
qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, &epSet); (void)qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, &epSet); // ignore error
*rsped = true; *rsped = true;
return; return;
} }
if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) {
QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY); QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY);
qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); (void)qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); // ignore error
*rsped = true; *rsped = true;
return; return;
} }
if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 9)) { if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 9)) {
qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); (void)qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); // ignore error
*rsped = true; *rsped = true;
return; return;
} }
@ -260,8 +273,8 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) {
if (++ignoreTime > 10 && 0 == taosRand() % 9) { if (++ignoreTime > 10 && 0 == taosRand() % 9) {
if (ctx->fetchMsgType == TDMT_SCH_FETCH) { if (ctx->fetchMsgType == TDMT_SCH_FETCH) {
qwBuildAndSendErrorRsp(TDMT_SCH_LINK_BROKEN, &ctx->ctrlConnInfo, TSDB_CODE_RPC_BROKEN_LINK); (void)qwBuildAndSendErrorRsp(TDMT_SCH_LINK_BROKEN, &ctx->ctrlConnInfo, TSDB_CODE_RPC_BROKEN_LINK); // ignore error
qwBuildAndSendErrorRsp(ctx->fetchMsgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); (void)qwBuildAndSendErrorRsp(ctx->fetchMsgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); // ignore error
*rsped = true; *rsped = true;
taosSsleep(3); taosSsleep(3);

View File

@ -21,7 +21,7 @@ int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **r
} }
if (NULL == *rsp) { if (NULL == *rsp) {
memset(pRsp, 0, sizeof(SRetrieveTableRsp)); TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp));
} }
*rsp = pRsp; *rsp = pRsp;
@ -105,8 +105,19 @@ int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SArray *pExecList) {
SExplainRsp rsp = {.numOfPlans = taosArrayGetSize(pExecList), .subplanInfo = pInfo}; SExplainRsp rsp = {.numOfPlans = taosArrayGetSize(pExecList), .subplanInfo = pInfo};
int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp); int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp);
if (contLen < 0) {
qError("tSerializeSExplainRsp failed, error: %x", terrno);
QW_RET(terrno);
}
void *pRsp = rpcMallocCont(contLen); void *pRsp = rpcMallocCont(contLen);
tSerializeSExplainRsp(pRsp, contLen, &rsp); if (NULL == pRsp) {
QW_RET(terrno);
}
contLen = tSerializeSExplainRsp(pRsp, contLen, &rsp);
if (contLen < 0) {
qError("tSerializeSExplainRsp second failed, error: %x", terrno);
QW_RET(terrno);
}
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.msgType = TDMT_SCH_EXPLAIN_RSP, .msgType = TDMT_SCH_EXPLAIN_RSP,
@ -123,8 +134,20 @@ int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SArray *pExecList) {
int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) { int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus); int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus);
if (contLen < 0) {
qError("tSerializeSSchedulerHbRsp failed, error: %x", terrno);
QW_RET(terrno);
}
void *pRsp = rpcMallocCont(contLen); void *pRsp = rpcMallocCont(contLen);
tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus); if (NULL == pRsp) {
QW_RET(terrno);
}
contLen = tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus);
if (contLen < 0) {
qError("tSerializeSSchedulerHbRsp second failed, error: %x", terrno);
QW_RET(terrno);
}
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.msgType = TDMT_SCH_QUERY_HEARTBEAT_RSP, .msgType = TDMT_SCH_QUERY_HEARTBEAT_RSP,
@ -143,7 +166,10 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve
int32_t code) { int32_t code) {
if (NULL == pRsp) { if (NULL == pRsp) {
pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
memset(pRsp, 0, sizeof(SRetrieveTableRsp)); if (NULL == pRsp) {
QW_RET(terrno);
}
TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp));
dataLength = 0; dataLength = 0;
} }
@ -164,6 +190,9 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve
#if 0 #if 0
int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code) { int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code) {
STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp)); STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp));
if (NULL == pRsp) {
QW_RET(terrno);
}
pRsp->code = code; pRsp->code = code;
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
@ -180,6 +209,9 @@ int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code) {
int32_t qwBuildAndSendDropRsp(SRpcHandleInfo *pConn, int32_t code) { int32_t qwBuildAndSendDropRsp(SRpcHandleInfo *pConn, int32_t code) {
STaskDropRsp *pRsp = (STaskDropRsp *)rpcMallocCont(sizeof(STaskDropRsp)); STaskDropRsp *pRsp = (STaskDropRsp *)rpcMallocCont(sizeof(STaskDropRsp));
if (NULL == pRsp) {
QW_RET(terrno);
}
pRsp->code = code; pRsp->code = code;
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
@ -405,6 +437,7 @@ int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) {
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
int32_t code = 0;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
SSubQueryMsg msg = {0}; SSubQueryMsg msg = {0};
if (tDeserializeSSubQueryMsg(pMsg->pCont, pMsg->contLen, &msg) < 0) { if (tDeserializeSSubQueryMsg(pMsg->pCont, pMsg->contLen, &msg) < 0) {
@ -419,8 +452,8 @@ int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) {
int32_t eId = msg.execId; int32_t eId = msg.execId;
QW_SCH_TASK_DLOG("Abort prerocessQuery start, handle:%p", pMsg->info.handle); QW_SCH_TASK_DLOG("Abort prerocessQuery start, handle:%p", pMsg->info.handle);
qwAbortPrerocessQuery(QW_FPARAMS()); code = qwAbortPrerocessQuery(QW_FPARAMS());
QW_SCH_TASK_DLOG("Abort prerocessQuery end, handle:%p", pMsg->info.handle); QW_SCH_TASK_DLOG("Abort prerocessQuery end, handle:%p, code:%x", pMsg->info.handle, code);
tFreeSSubQueryMsg(&msg); tFreeSSubQueryMsg(&msg);
@ -435,7 +468,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
int32_t code = 0; int32_t code = 0;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE); QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.queryProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.queryProcessed, 1);
SSubQueryMsg msg = {0}; SSubQueryMsg msg = {0};
@ -477,7 +510,7 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
SQWTaskCtx *handles = NULL; SQWTaskCtx *handles = NULL;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE); QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.cqueryProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.cqueryProcessed, 1);
if (NULL == msg || pMsg->contLen < sizeof(*msg)) { if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
@ -495,9 +528,9 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
QW_SCH_TASK_DLOG("processCQuery start, node:%p, handle:%p", node, pMsg->info.handle); QW_SCH_TASK_DLOG("processCQuery start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessCQuery(QW_FPARAMS(), &qwMsg)); code = qwProcessCQuery(QW_FPARAMS(), &qwMsg);
QW_SCH_TASK_DLOG("processCQuery end, node:%p", node); QW_SCH_TASK_DLOG("processCQuery end, node:%p, code:0x%x", node, code);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -510,7 +543,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
SResFetchReq req = {0}; SResFetchReq req = {0};
SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.fetchProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.fetchProcessed, 1);
if (tDeserializeSResFetchReq(pMsg->pCont, pMsg->contLen, &req) < 0) { if (tDeserializeSResFetchReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
@ -528,9 +561,9 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
QW_SCH_TASK_DLOG("processFetch start, node:%p, handle:%p", node, pMsg->info.handle); QW_SCH_TASK_DLOG("processFetch start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessFetch(QW_FPARAMS(), &qwMsg)); int32_t code = qwProcessFetch(QW_FPARAMS(), &qwMsg);
QW_SCH_TASK_DLOG("processFetch end, node:%p", node); QW_SCH_TASK_DLOG("processFetch end, node:%p, code:%x", node, code);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -538,7 +571,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
int32_t qWorkerProcessRspMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { int32_t qWorkerProcessRspMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) {
SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
if (mgmt) { if (mgmt) {
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.rspProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.rspProcessed, 1);
} }
@ -557,7 +590,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
int32_t code = 0; int32_t code = 0;
STaskCancelReq *msg = pMsg->pCont; STaskCancelReq *msg = pMsg->pCont;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.cancelProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.cancelProcessed, 1);
if (NULL == msg || pMsg->contLen < sizeof(*msg)) { if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
@ -598,7 +631,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6
int32_t code = 0; int32_t code = 0;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.dropProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.dropProcessed, 1);
STaskDropReq msg = {0}; STaskDropReq msg = {0};
@ -621,9 +654,9 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6
QW_SCH_TASK_DLOG("processDrop start, node:%p, handle:%p", node, pMsg->info.handle); QW_SCH_TASK_DLOG("processDrop start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessDrop(QW_FPARAMS(), &qwMsg)); code = qwProcessDrop(QW_FPARAMS(), &qwMsg);
QW_SCH_TASK_DLOG("processDrop end, node:%p", node); QW_SCH_TASK_DLOG("processDrop end, node:%p, code:%x", node, code);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -636,7 +669,7 @@ int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
int32_t code = 0; int32_t code = 0;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.notifyProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.notifyProcessed, 1);
STaskNotifyReq msg = {0}; STaskNotifyReq msg = {0};
@ -655,9 +688,9 @@ int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
QW_SCH_TASK_DLOG("processNotify start, node:%p, handle:%p", node, pMsg->info.handle); QW_SCH_TASK_DLOG("processNotify start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessNotify(QW_FPARAMS(), &qwMsg)); code = qwProcessNotify(QW_FPARAMS(), &qwMsg);
QW_SCH_TASK_DLOG("processNotify end, node:%p", node); QW_SCH_TASK_DLOG("processNotify end, node:%p, code:%x", node, code);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -672,7 +705,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_
SSchedulerHbReq req = {0}; SSchedulerHbReq req = {0};
SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.hbProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.hbProcessed, 1);
if (NULL == pMsg->pCont) { if (NULL == pMsg->pCont) {
@ -694,9 +727,9 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_
QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->info.handle); QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessHb(mgmt, &qwMsg, &req)); code = qwProcessHb(mgmt, &qwMsg, &req);
QW_SCH_DLOG("processHb end, node:%p", node); QW_SCH_DLOG("processHb end, node:%p, code:%x", node, code);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -712,7 +745,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD
QW_STAT_INC(mgmt->stat.msgStat.deleteProcessed, 1); QW_STAT_INC(mgmt->stat.msgStat.deleteProcessed, 1);
tDeserializeSVDeleteReq(pMsg->pCont, pMsg->contLen, &req); QW_ERR_RET(tDeserializeSVDeleteReq(pMsg->pCont, pMsg->contLen, &req));
uint64_t sId = req.sId; uint64_t sId = req.sId;
uint64_t qId = req.queryId; uint64_t qId = req.queryId;

View File

@ -323,34 +323,52 @@ static void freeExplainExecItem(void *param) {
int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
int32_t code = TSDB_CODE_SUCCESS;
qTaskInfo_t taskHandle = ctx->taskHandle; qTaskInfo_t taskHandle = ctx->taskHandle;
ctx->explainRsped = true; ctx->explainRsped = true;
SArray *execInfoList = taosArrayInit(4, sizeof(SExplainExecInfo)); SArray *execInfoList = taosArrayInit(4, sizeof(SExplainExecInfo));
QW_ERR_RET(qGetExplainExecInfo(taskHandle, execInfoList)); if (NULL == execInfoList) {
QW_ERR_JRET(terrno);
}
QW_ERR_JRET(qGetExplainExecInfo(taskHandle, execInfoList));
if (ctx->localExec) { if (ctx->localExec) {
SExplainLocalRsp localRsp = {0}; SExplainLocalRsp localRsp = {0};
localRsp.rsp.numOfPlans = taosArrayGetSize(execInfoList); localRsp.rsp.numOfPlans = taosArrayGetSize(execInfoList);
SExplainExecInfo *pExec = taosMemoryCalloc(localRsp.rsp.numOfPlans, sizeof(SExplainExecInfo)); SExplainExecInfo *pExec = taosMemoryCalloc(localRsp.rsp.numOfPlans, sizeof(SExplainExecInfo));
memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); if (NULL == pExec) {
QW_ERR_JRET(terrno);
}
(void)memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo));
localRsp.rsp.subplanInfo = pExec; localRsp.rsp.subplanInfo = pExec;
localRsp.qId = qId; localRsp.qId = qId;
localRsp.tId = tId; localRsp.tId = tId;
localRsp.rId = rId; localRsp.rId = rId;
localRsp.eId = eId; localRsp.eId = eId;
taosArrayPush(ctx->explainRes, &localRsp); if (NULL == taosArrayPush(ctx->explainRes, &localRsp)) {
QW_ERR_JRET(terrno);
}
taosArrayDestroy(execInfoList); taosArrayDestroy(execInfoList);
execInfoList = NULL;
} else { } else {
SRpcHandleInfo connInfo = ctx->ctrlConnInfo; SRpcHandleInfo connInfo = ctx->ctrlConnInfo;
connInfo.ahandle = NULL; connInfo.ahandle = NULL;
int32_t code = qwBuildAndSendExplainRsp(&connInfo, execInfoList); int32_t code = qwBuildAndSendExplainRsp(&connInfo, execInfoList);
taosArrayDestroyEx(execInfoList, freeExplainExecItem); taosArrayDestroyEx(execInfoList, freeExplainExecItem);
execInfoList = NULL;
QW_ERR_RET(code); QW_ERR_RET(code);
} }
return TSDB_CODE_SUCCESS; _return:
taosArrayDestroyEx(execInfoList, freeExplainExecItem);
return code;
} }
@ -503,37 +521,48 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) {
*pParam = &gQwMgmt.param[paramIdx]; *pParam = &gQwMgmt.param[paramIdx];
} }
void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) {
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN]; char tbName[TSDB_TABLE_NAME_LEN];
STbVerInfo tbInfo; STbVerInfo tbInfo;
int32_t i = 0; int32_t i = 0;
int32_t code = TSDB_CODE_SUCCESS;
bool tbGet = false;
while (true) { while (true) {
if (qGetQueryTableSchemaVersion(pTaskInfo, dbFName, tbName, &tbInfo.sversion, &tbInfo.tversion, i) < 0) { tbGet = false;
code = qGetQueryTableSchemaVersion(pTaskInfo, dbFName, tbName, &tbInfo.sversion, &tbInfo.tversion, i, &tbGet);
if (TSDB_CODE_SUCCESS != code || !tbGet) {
break; break;
} }
if (dbFName[0] && tbName[0]) { if (dbFName[0] && tbName[0]) {
sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName); (void)sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName);
} else { } else {
tbInfo.tbFName[0] = 0; tbInfo.tbFName[0] = 0;
} }
if (NULL == ctx->tbInfo) { if (NULL == ctx->tbInfo) {
ctx->tbInfo = taosArrayInit(1, sizeof(tbInfo)); ctx->tbInfo = taosArrayInit(1, sizeof(tbInfo));
if (NULL == ctx->tbInfo) {
QW_ERR_RET(terrno);
}
} }
taosArrayPush(ctx->tbInfo, &tbInfo); if (NULL == taosArrayPush(ctx->tbInfo, &tbInfo)) {
QW_ERR_RET(terrno);
}
i++; i++;
} }
QW_RET(code);
} }
void qwCloseRef(void) { void qwCloseRef(void) {
taosWLockLatch(&gQwMgmt.lock); taosWLockLatch(&gQwMgmt.lock);
if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) { if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) {
taosCloseRef(gQwMgmt.qwRef); (void)taosCloseRef(gQwMgmt.qwRef); // ignore error
gQwMgmt.qwRef = -1; gQwMgmt.qwRef = -1;
} }
taosWUnLockLatch(&gQwMgmt.lock); taosWUnLockLatch(&gQwMgmt.lock);
@ -550,7 +579,7 @@ void qwDestroyImpl(void *pMgmt) {
int32_t schStatusCount = 0; int32_t schStatusCount = 0;
qDebug("start to destroy qworker, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt); qDebug("start to destroy qworker, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt);
taosTmrStop(mgmt->hbTimer); (void)taosTmrStop(mgmt->hbTimer); //ignore error
mgmt->hbTimer = NULL; mgmt->hbTimer = NULL;
taosTmrCleanUp(mgmt->timer); taosTmrCleanUp(mgmt->timer);
@ -584,7 +613,7 @@ void qwDestroyImpl(void *pMgmt) {
taosMemoryFree(mgmt); taosMemoryFree(mgmt);
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
qwCloseRef(); qwCloseRef();
@ -641,24 +670,33 @@ int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type) {
return pStat->num ? (pStat->total / pStat->num) : 0; return pStat->num ? (pStat->total / pStat->num) : 0;
default: default:
qError("unsupported queue type %d", type); qError("unsupported queue type %d", type);
break;
} }
return -1; return -1;
} }
void qwClearExpiredSch(SQWorker *mgmt, SArray *pExpiredSch) { void qwClearExpiredSch(SQWorker *mgmt, SArray *pExpiredSch) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t num = taosArrayGetSize(pExpiredSch); int32_t num = taosArrayGetSize(pExpiredSch);
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
uint64_t *sId = taosArrayGet(pExpiredSch, i); uint64_t *sId = taosArrayGet(pExpiredSch, i);
SQWSchStatus *pSch = NULL; SQWSchStatus *pSch = NULL;
if (qwAcquireScheduler(mgmt, *sId, QW_WRITE, &pSch)) { if (NULL == sId) {
qError("get the %dth sch failed, code:%x", i, terrno);
break;
}
code = qwAcquireScheduler(mgmt, *sId, QW_WRITE, &pSch);
if (TSDB_CODE_SUCCESS != code) {
qError("acquire sch %" PRIx64 " failed, code:%x", *sId, code);
continue; continue;
} }
if (taosHashGetSize(pSch->tasksHash) <= 0) { if (taosHashGetSize(pSch->tasksHash) <= 0) {
qwDestroySchStatus(pSch); qwDestroySchStatus(pSch);
taosHashRemove(mgmt->schHash, sId, sizeof(*sId)); code = taosHashRemove(mgmt->schHash, sId, sizeof(*sId));
qDebug("sch %" PRIx64 " destroyed", *sId); qDebug("sch %" PRIx64 " destroy result code:%x", *sId, code);
} }
qwReleaseScheduler(QW_WRITE, mgmt); qwReleaseScheduler(QW_WRITE, mgmt);

View File

@ -18,10 +18,11 @@ SQWorkerMgmt gQwMgmt = {
.qwNum = 0, .qwNum = 0,
}; };
int32_t qwStopAllTasks(SQWorker *mgmt) { void qwStopAllTasks(SQWorker *mgmt) {
uint64_t qId, tId, sId; uint64_t qId, tId, sId;
int32_t eId; int32_t eId;
int64_t rId = 0; int64_t rId = 0;
int32_t code = TSDB_CODE_SUCCESS;
void *pIter = taosHashIterate(mgmt->ctxHash, NULL); void *pIter = taosHashIterate(mgmt->ctxHash, NULL);
while (pIter) { while (pIter) {
@ -44,22 +45,29 @@ int32_t qwStopAllTasks(SQWorker *mgmt) {
} }
if (QW_QUERY_RUNNING(ctx)) { if (QW_QUERY_RUNNING(ctx)) {
qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED); code = qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED);
QW_TASK_DLOG_E("task running, async killed"); if (TSDB_CODE_SUCCESS != code) {
QW_TASK_ELOG("task running, async kill failed, error: %x", code);
} else {
QW_TASK_DLOG_E("task running, async killed");
}
} else if (QW_FETCH_RUNNING(ctx)) { } else if (QW_FETCH_RUNNING(ctx)) {
QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED); QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED);
QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP);
QW_TASK_DLOG_E("task fetching, update drop received"); QW_TASK_DLOG_E("task fetching, update drop received");
} else { } else {
qwDropTask(QW_FPARAMS()); code = qwDropTask(QW_FPARAMS());
if (TSDB_CODE_SUCCESS != code) {
QW_TASK_ELOG("task drop failed, error: %x", code);
} else {
QW_TASK_DLOG_E("task dropped");
}
} }
QW_UNLOCK(QW_WRITE, &ctx->lock); QW_UNLOCK(QW_WRITE, &ctx->lock);
pIter = taosHashIterate(mgmt->ctxHash, pIter); pIter = taosHashIterate(mgmt->ctxHash, pIter);
} }
return TSDB_CODE_SUCCESS;
} }
int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
@ -111,7 +119,7 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) { int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) {
if ((!quickRsp) || QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy) { if ((!quickRsp) || QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy) {
if (!ctx->localExec) { if (!ctx->localExec) {
qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx); QW_ERR_RET(qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx));
QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode)); QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode));
} }
@ -140,6 +148,10 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
} }
SArray *pResList = taosArrayInit(4, POINTER_BYTES); SArray *pResList = taosArrayInit(4, POINTER_BYTES);
if (NULL == pResList) {
QW_ERR_RET(terrno);
}
while (true) { while (true) {
QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); QW_TASK_DLOG("start to execTask, loopIdx:%d", i++);
@ -165,6 +177,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
size_t numOfResBlock = taosArrayGetSize(pResList); size_t numOfResBlock = taosArrayGetSize(pResList);
for (int32_t j = 0; j < numOfResBlock; ++j) { for (int32_t j = 0; j < numOfResBlock; ++j) {
SSDataBlock *pRes = taosArrayGetP(pResList, j); SSDataBlock *pRes = taosArrayGetP(pResList, j);
if (NULL == pRes) {
QW_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SInputData inputData = {.pData = pRes}; SInputData inputData = {.pData = pRes};
code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue);
@ -226,7 +241,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
} }
_return: _return:
taosArrayDestroy(pResList); taosArrayDestroy(pResList);
QW_RET(code); QW_RET(code);
} }
@ -241,7 +258,8 @@ bool qwTaskNotInExec(SQWTaskCtx *ctx) {
int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) {
int32_t taskNum = 0; int32_t taskNum = 0;
int32_t code = TSDB_CODE_SUCCESS;
hbInfo->connInfo = sch->hbConnInfo; hbInfo->connInfo = sch->hbConnInfo;
hbInfo->rsp.epId = sch->hbEpId; hbInfo->rsp.epId = sch->hbEpId;
@ -272,7 +290,11 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo)
status.status = taskStatus->status; status.status = taskStatus->status;
status.refId = taskStatus->refId; status.refId = taskStatus->refId;
taosArrayPush(hbInfo->rsp.taskStatus, &status); if (NULL == taosArrayPush(hbInfo->rsp.taskStatus, &status)) {
taosHashCancelIterate(sch->tasksHash, pIter);
code = terrno;
break;
}
++i; ++i;
pIter = taosHashIterate(sch->tasksHash, pIter); pIter = taosHashIterate(sch->tasksHash, pIter);
@ -280,7 +302,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo)
QW_UNLOCK(QW_READ, &sch->tasksLock); QW_UNLOCK(QW_READ, &sch->tasksLock);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, int32_t *pRawDataLen, void **rspMsg, int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, int32_t *pRawDataLen, void **rspMsg,
@ -320,7 +342,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
pOutput->numOfRows); pOutput->numOfRows);
if (!ctx->dynamicTask) { if (!ctx->dynamicTask) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask); QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask));
} }
if (NULL == pRsp) { if (NULL == pRsp) {
@ -375,7 +397,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) {
QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %" PRId64, pOutput->numOfBlocks, QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %" PRId64, pOutput->numOfBlocks,
pOutput->numOfRows); pOutput->numOfRows);
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask); QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask));
break; break;
} }
@ -430,8 +452,8 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes
pRes->skey = pDelRes->skey; pRes->skey = pDelRes->skey;
pRes->ekey = pDelRes->ekey; pRes->ekey = pDelRes->ekey;
pRes->affectedRows = pDelRes->affectedRows; pRes->affectedRows = pDelRes->affectedRows;
strcpy(pRes->tableFName, pDelRes->tableName); TAOS_STRCPY(pRes->tableFName, pDelRes->tableName);
strcpy(pRes->tsColName, pDelRes->tsColName); TAOS_STRCPY(pRes->tsColName, pDelRes->tsColName);
taosMemoryFree(output.pData); taosMemoryFree(output.pData);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -464,7 +486,7 @@ int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg, int32
qwMsg->connInfo = ctx->dataConnInfo; qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code); QW_ERR_RET(qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code));
rsp = NULL; rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
@ -650,7 +672,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
_return: _return:
if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) { if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC, ctx->dynamicTask); code = qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC, ctx->dynamicTask);
ctx->queryGotData = true; ctx->queryGotData = true;
} }
@ -660,7 +682,10 @@ _return:
qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); qwDbgSimulateRedirect(&qwMsg, ctx, &rsped);
qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped);
if (!rsped) { if (!rsped) {
qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false); int32_t newCode = qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false);
if (TSDB_CODE_SUCCESS != newCode && TSDB_CODE_SUCCESS == code) {
code = newCode;
}
} }
} }
@ -672,7 +697,7 @@ _return:
} }
if (code) { if (code) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); (void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); // already in error, ignore new error
} }
QW_UNLOCK(QW_WRITE, &ctx->lock); QW_UNLOCK(QW_WRITE, &ctx->lock);
@ -687,11 +712,11 @@ _return:
int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF) { int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF) {
QW_ERR_RET(qwDropTask(QW_FPARAMS())); QW_ERR_RET(qwDropTask(QW_FPARAMS()));
QW_RET(TSDB_CODE_SUCCESS); return TSDB_CODE_SUCCESS;
} }
int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
int32_t code = 0; int32_t code = TSDB_CODE_SUCCESS;
SQWTaskCtx *ctx = NULL; SQWTaskCtx *ctx = NULL;
QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo));
@ -706,7 +731,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT));
qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); QW_ERR_JRET(qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true));
_return: _return:
@ -715,7 +740,7 @@ _return:
qwReleaseTaskCtx(mgmt, ctx); qwReleaseTaskCtx(mgmt, ctx);
} }
QW_RET(TSDB_CODE_SUCCESS); return TSDB_CODE_SUCCESS;
} }
int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
@ -761,7 +786,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->taskHandle, pTaskInfo);
atomic_store_ptr(&ctx->sinkHandle, sinkHandle); atomic_store_ptr(&ctx->sinkHandle, sinkHandle);
qwSaveTbVersionInfo(pTaskInfo, ctx); QW_ERR_JRET(qwSaveTbVersionInfo(pTaskInfo, ctx));
if (!ctx->dynamicTask) { if (!ctx->dynamicTask) {
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL)); QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL));
@ -778,7 +803,7 @@ _return:
input.msgType = qwMsg->msgType; input.msgType = qwMsg->msgType;
code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL);
qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code); QW_ERR_RET(qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code));
QW_RET(TSDB_CODE_SUCCESS); QW_RET(TSDB_CODE_SUCCESS);
} }
@ -829,7 +854,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
qwMsg->connInfo = ctx->dataConnInfo; qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code); QW_ERR_JRET(qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code));
rsp = NULL; rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code,
@ -851,9 +876,14 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
rsp = NULL; rsp = NULL;
qwMsg->connInfo = ctx->dataConnInfo; qwMsg->connInfo = ctx->dataConnInfo;
qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, NULL, 0, code); code = qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, NULL, 0, code);
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), if (TSDB_CODE_SUCCESS != code) {
0); QW_TASK_ELOG("fetch rsp send fail, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
0);
} else {
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
0);
}
} }
QW_LOCK(QW_WRITE, &ctx->lock); QW_LOCK(QW_WRITE, &ctx->lock);
@ -869,7 +899,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
} while (true); } while (true);
input.code = code; input.code = code;
qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL); QW_ERR_RET(qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL));
QW_RET(TSDB_CODE_SUCCESS); QW_RET(TSDB_CODE_SUCCESS);
} }
@ -922,7 +952,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
} else if (QW_QUERY_RUNNING(ctx)) { } else if (QW_QUERY_RUNNING(ctx)) {
atomic_store_8((int8_t *)&ctx->queryContinue, 1); atomic_store_8((int8_t *)&ctx->queryContinue, 1);
} else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) { } else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC, ctx->dynamicTask); QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC, ctx->dynamicTask));
atomic_store_8((int8_t *)&ctx->queryInQueue, 1); atomic_store_8((int8_t *)&ctx->queryInQueue, 1);
QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), &qwMsg->connInfo)); QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), &qwMsg->connInfo));
@ -952,9 +982,14 @@ _return:
} }
if (!rsped) { if (!rsped) {
qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); code = qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
QW_TASK_DLOG("fetch rsp send, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), if (TSDB_CODE_SUCCESS != code) {
qwMsg->connInfo.handle, code, tstrerror(code), dataLen); QW_TASK_ELOG("fetch rsp send fail, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1),
qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
} else {
QW_TASK_DLOG("fetch rsp send, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1),
qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
}
} else { } else {
qwFreeFetchRsp(rsp); qwFreeFetchRsp(rsp);
rsp = NULL; rsp = NULL;
@ -985,7 +1020,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
if (QW_QUERY_RUNNING(ctx)) { if (QW_QUERY_RUNNING(ctx)) {
QW_ERR_JRET(qwKillTaskHandle(ctx, TSDB_CODE_TSC_QUERY_CANCELLED)); QW_ERR_JRET(qwKillTaskHandle(ctx, TSDB_CODE_TSC_QUERY_CANCELLED));
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP, ctx->dynamicTask); QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP, ctx->dynamicTask));
} else { } else {
QW_ERR_JRET(qwDropTask(QW_FPARAMS())); QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
dropped = true; dropped = true;
@ -1001,7 +1036,7 @@ _return:
if (code) { if (code) {
if (ctx) { if (ctx) {
QW_UPDATE_RSP_CODE(ctx, code); QW_UPDATE_RSP_CODE(ctx, code);
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); (void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); // task already failed, no more error handling
} else { } else {
tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER); tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER);
} }
@ -1035,7 +1070,7 @@ int32_t qwProcessNotify(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
if (QW_QUERY_RUNNING(ctx)) { if (QW_QUERY_RUNNING(ctx)) {
QW_ERR_JRET(qwKillTaskHandle(ctx, TSDB_CODE_TSC_QUERY_CANCELLED)); QW_ERR_JRET(qwKillTaskHandle(ctx, TSDB_CODE_TSC_QUERY_CANCELLED));
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask); QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask));
} }
switch (qwMsg->msgType) { switch (qwMsg->msgType) {
@ -1055,7 +1090,7 @@ _return:
if (code) { if (code) {
if (ctx) { if (ctx) {
QW_UPDATE_RSP_CODE(ctx, code); QW_UPDATE_RSP_CODE(ctx, code);
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); (void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); // task already failed, no more error handling
} }
} }
@ -1091,8 +1126,8 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
sch->hbConnInfo.handle = NULL; sch->hbConnInfo.handle = NULL;
} }
memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); (void)memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo));
memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); (void)memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId));
QW_UNLOCK(QW_WRITE, &sch->hbConnLock); QW_UNLOCK(QW_WRITE, &sch->hbConnLock);
@ -1103,8 +1138,8 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
_return: _return:
memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); (void)memcpy(&rsp.epId, &req->epId, sizeof(req->epId));
qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code); code = qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code);
if (code) { if (code) {
tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER); tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER);
@ -1125,7 +1160,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
int64_t refId = hbParam->refId; int64_t refId = hbParam->refId;
SQWorker *mgmt = qwAcquire(refId); SQWorker *mgmt = qwAcquire(refId);
if (NULL == mgmt) { if (NULL == mgmt) {
QW_DLOG("qwAcquire %" PRIx64 "failed", refId); QW_DLOG("qwAcquire %" PRIx64 "failed, code:0x%x", refId, terrno);
return; return;
} }
@ -1137,7 +1172,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
qwDbgDumpMgmtInfo(mgmt); qwDbgDumpMgmtInfo(mgmt);
if (gQWDebug.forceStop) { if (gQWDebug.forceStop) {
(void)qwStopAllTasks(mgmt); qwStopAllTasks(mgmt);
} }
QW_LOCK(QW_READ, &mgmt->schLock); QW_LOCK(QW_READ, &mgmt->schLock);
@ -1145,8 +1180,8 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
int32_t schNum = taosHashGetSize(mgmt->schHash); int32_t schNum = taosHashGetSize(mgmt->schHash);
if (schNum <= 0) { if (schNum <= 0) {
QW_UNLOCK(QW_READ, &mgmt->schLock); QW_UNLOCK(QW_READ, &mgmt->schLock);
taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); (void)taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); // ignore error
qwRelease(refId); (void)qwRelease(refId); // ignore error
return; return;
} }
@ -1156,9 +1191,9 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
QW_UNLOCK(QW_READ, &mgmt->schLock); QW_UNLOCK(QW_READ, &mgmt->schLock);
taosMemoryFree(rspList); taosMemoryFree(rspList);
taosArrayDestroy(pExpiredSch); taosArrayDestroy(pExpiredSch);
QW_ELOG("calloc %d SQWHbInfo failed", schNum); QW_ELOG("calloc %d SQWHbInfo failed, code:%x", schNum, terrno);
taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); (void)taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); // ignore error
qwRelease(refId); (void)qwRelease(refId); // ignore error
return; return;
} }
@ -1174,7 +1209,11 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
if (sch1->hbBrokenTs > 0 && ((currentMs - sch1->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) && if (sch1->hbBrokenTs > 0 && ((currentMs - sch1->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) &&
taosHashGetSize(sch1->tasksHash) <= 0) { taosHashGetSize(sch1->tasksHash) <= 0) {
taosArrayPush(pExpiredSch, sId); if (NULL == taosArrayPush(pExpiredSch, sId)) {
QW_ELOG("add sId 0x%" PRIx64 " to expiredSch failed, code:%x", *sId, terrno);
taosHashCancelIterate(mgmt->schHash, pIter);
break;
}
} }
pIter = taosHashIterate(mgmt->schHash, pIter); pIter = taosHashIterate(mgmt->schHash, pIter);
@ -1196,7 +1235,7 @@ _return:
QW_UNLOCK(QW_READ, &mgmt->schLock); QW_UNLOCK(QW_READ, &mgmt->schLock);
for (int32_t j = 0; j < i; ++j) { for (int32_t j = 0; j < i; ++j) {
qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); (void)qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); // ignore error
/*QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),*/ /*QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),*/
/*(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));*/ /*(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));*/
tFreeSSchedulerHbRsp(&rspList[j].rsp); tFreeSSchedulerHbRsp(&rspList[j].rsp);
@ -1209,8 +1248,8 @@ _return:
taosMemoryFreeClear(rspList); taosMemoryFreeClear(rspList);
taosArrayDestroy(pExpiredSch); taosArrayDestroy(pExpiredSch);
taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); (void)taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); // ignore error
qwRelease(refId); (void)qwRelease(refId); // ignore error
} }
int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) {
@ -1260,19 +1299,19 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S
int32_t qwNum = atomic_add_fetch_32(&gQwMgmt.qwNum, 1); int32_t qwNum = atomic_add_fetch_32(&gQwMgmt.qwNum, 1);
if (1 == qwNum) { if (1 == qwNum) {
memset(gQwMgmt.param, 0, sizeof(gQwMgmt.param)); TAOS_MEMSET(gQwMgmt.param, 0, sizeof(gQwMgmt.param));
} }
int32_t code = qwOpenRef(); int32_t code = qwOpenRef();
if (code) { if (code) {
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
QW_RET(code); QW_RET(code);
} }
SQWorker *mgmt = taosMemoryCalloc(1, sizeof(SQWorker)); SQWorker *mgmt = taosMemoryCalloc(1, sizeof(SQWorker));
if (NULL == mgmt) { if (NULL == mgmt) {
qError("calloc %d failed", (int32_t)sizeof(SQWorker)); qError("calloc %d failed", (int32_t)sizeof(SQWorker));
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
QW_RET(TSDB_CODE_OUT_OF_MEMORY); QW_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
@ -1306,7 +1345,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S
if (pMsgCb) { if (pMsgCb) {
mgmt->msgCb = *pMsgCb; mgmt->msgCb = *pMsgCb;
} else { } else {
memset(&mgmt->msgCb, 0, sizeof(mgmt->msgCb)); TAOS_MEMSET(&mgmt->msgCb, 0, sizeof(mgmt->msgCb));
} }
mgmt->refId = taosAddRef(gQwMgmt.qwRef, mgmt); mgmt->refId = taosAddRef(gQwMgmt.qwRef, mgmt);
@ -1333,14 +1372,14 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S
_return: _return:
if (mgmt->refId >= 0) { if (mgmt->refId >= 0) {
qwRelease(mgmt->refId); (void)qwRelease(mgmt->refId); // ignore error
} else { } else {
taosHashCleanup(mgmt->schHash); taosHashCleanup(mgmt->schHash);
taosHashCleanup(mgmt->ctxHash); taosHashCleanup(mgmt->ctxHash);
taosTmrCleanUp(mgmt->timer); taosTmrCleanUp(mgmt->timer);
taosMemoryFreeClear(mgmt); taosMemoryFreeClear(mgmt);
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); (void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
} }
QW_RET(code); QW_RET(code);
@ -1353,7 +1392,7 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) {
atomic_store_8(&mgmt->nodeStopped, 1); atomic_store_8(&mgmt->nodeStopped, 1);
(void)qwStopAllTasks(mgmt); qwStopAllTasks(mgmt);
} }
void qWorkerDestroy(void **qWorkerMgmt) { void qWorkerDestroy(void **qWorkerMgmt) {
@ -1383,7 +1422,7 @@ int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pSt
SQWorker *mgmt = (SQWorker *)qWorkerMgmt; SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
SDataSinkStat sinkStat = {0}; SDataSinkStat sinkStat = {0};
dsDataSinkGetCacheSize(&sinkStat); QW_ERR_RET(dsDataSinkGetCacheSize(&sinkStat));
pStat->cacheDataSize = sinkStat.cachedSize; pStat->cacheDataSize = sinkStat.cachedSize;
pStat->queryProcessed = QW_STAT_GET(mgmt->stat.msgStat.queryProcessed); pStat->queryProcessed = QW_STAT_GET(mgmt->stat.msgStat.queryProcessed);
@ -1427,6 +1466,10 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64
ctx->explainRes = explainRes; ctx->explainRes = explainRes;
rHandle.pMsgCb = taosMemoryCalloc(1, sizeof(SMsgCb)); rHandle.pMsgCb = taosMemoryCalloc(1, sizeof(SMsgCb));
if (NULL == rHandle.pMsgCb) {
QW_ERR_JRET(terrno);
}
rHandle.pMsgCb->clientRpc = qwMsg->connInfo.handle; rHandle.pMsgCb->clientRpc = qwMsg->connInfo.handle;
code = qCreateExecTask(&rHandle, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH); code = qCreateExecTask(&rHandle, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH);

View File

@ -130,30 +130,32 @@ void qwtBuildFetchReqMsg(SResFetchReq *fetchMsg, SRpcMsg *fetchRpc) {
fetchRpc->contLen = sizeof(SResFetchReq); fetchRpc->contLen = sizeof(SResFetchReq);
} }
void qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) { int qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) {
dropMsg->sId = 1; dropMsg->sId = 1;
dropMsg->queryId = atomic_load_64(&qwtTestQueryId); dropMsg->queryId = atomic_load_64(&qwtTestQueryId);
dropMsg->taskId = 1; dropMsg->taskId = 1;
int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, dropMsg); int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, dropMsg);
if (msgSize < 0) { if (msgSize < 0) {
return; return terrno;
} }
char *msg = (char*)taosMemoryCalloc(1, msgSize); char *msg = (char*)taosMemoryCalloc(1, msgSize);
if (NULL == msg) { if (NULL == msg) {
return; return terrno;
} }
if (tSerializeSTaskDropReq(msg, msgSize, dropMsg) < 0) { if (tSerializeSTaskDropReq(msg, msgSize, dropMsg) < 0) {
taosMemoryFree(msg); taosMemoryFree(msg);
return; return terrno;
} }
dropRpc->msgType = TDMT_SCH_DROP_TASK; dropRpc->msgType = TDMT_SCH_DROP_TASK;
dropRpc->pCont = msg; dropRpc->pCont = msg;
dropRpc->contLen = msgSize; dropRpc->contLen = msgSize;
return TSDB_CODE_SUCCESS;
} }
int32_t qwtStringToPlan(const char *str, SSubplan **subplan) { int32_t qwtStringToPlan(const char *str, SSubplan **subplan) {
@ -164,7 +166,11 @@ int32_t qwtStringToPlan(const char *str, SSubplan **subplan) {
int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) {
taosWLockLatch(&qwtTestFetchQueueLock); taosWLockLatch(&qwtTestFetchQueueLock);
struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg)); struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg));
memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); if (NULL == newMsg) {
printf("malloc failed");
assert(0);
}
(void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg; qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg;
if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) { if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) {
qwtTestFetchQueueWIdx = 0; qwtTestFetchQueueWIdx = 0;
@ -178,7 +184,10 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) {
} }
taosWUnLockLatch(&qwtTestFetchQueueLock); taosWUnLockLatch(&qwtTestFetchQueueLock);
tsem_post(&qwtTestFetchSem); if (tsem_post(&qwtTestFetchSem) < 0) {
printf("tsem_post failed, errno:%d", errno);
assert(0);
}
return 0; return 0;
} }
@ -186,7 +195,11 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) {
int32_t qwtPutReqToQueue(void *node, EQueueType qtype, struct SRpcMsg *pMsg) { int32_t qwtPutReqToQueue(void *node, EQueueType qtype, struct SRpcMsg *pMsg) {
taosWLockLatch(&qwtTestQueryQueueLock); taosWLockLatch(&qwtTestQueryQueueLock);
struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg)); struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg));
memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); if (NULL == newMsg) {
printf("malloc failed");
assert(0);
}
(void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg; qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg;
if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) {
qwtTestQueryQueueWIdx = 0; qwtTestQueryQueueWIdx = 0;
@ -200,22 +213,34 @@ int32_t qwtPutReqToQueue(void *node, EQueueType qtype, struct SRpcMsg *pMsg) {
} }
taosWUnLockLatch(&qwtTestQueryQueueLock); taosWUnLockLatch(&qwtTestQueryQueueLock);
tsem_post(&qwtTestQuerySem); if (tsem_post(&qwtTestQuerySem) < 0) {
printf("tsem_post failed, errno:%d", errno);
assert(0);
}
return 0; return 0;
} }
void qwtSendReqToDnode(void *pVnode, struct SEpSet *epSet, struct SRpcMsg *pReq) {} void qwtSendReqToDnode(void *pVnode, struct SEpSet *epSet, struct SRpcMsg *pReq) {}
void qwtRpcSendResponse(const SRpcMsg *pRsp) { int qwtRpcSendResponse(const SRpcMsg *pRsp) {
int32_t code = 0;
switch (pRsp->msgType) { switch (pRsp->msgType) {
case TDMT_SCH_QUERY_RSP: case TDMT_SCH_QUERY_RSP:
case TDMT_SCH_MERGE_QUERY_RSP: { case TDMT_SCH_MERGE_QUERY_RSP: {
SQueryTableRsp *rsp = (SQueryTableRsp *)pRsp->pCont; SQueryTableRsp *rsp = (SQueryTableRsp *)pRsp->pCont;
if (pRsp->code) { if (pRsp->code) {
qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); code = qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc);
qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); if (code) {
assert(0);
return code;
}
code = qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc);
if (code) {
assert(0);
return code;
}
} }
rpcFreeCont(rsp); rpcFreeCont(rsp);
@ -227,13 +252,25 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) {
if (0 == pRsp->code && 0 == rsp->completed) { if (0 == pRsp->code && 0 == rsp->completed) {
qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc); qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc);
qwtPutReqToFetchQueue((void *)0x1, &qwtfetchRpc); code = qwtPutReqToFetchQueue((void *)0x1, &qwtfetchRpc);
if (code) {
assert(0);
return code;
}
rpcFreeCont(rsp); rpcFreeCont(rsp);
return; return code;
} }
qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc); code = qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc);
qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc); if (code) {
assert(0);
return code;
}
code = qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc);
if (code) {
assert(0);
return code;
}
rpcFreeCont(rsp); rpcFreeCont(rsp);
break; break;
@ -245,9 +282,11 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) {
qwtTestCaseFinished = true; qwtTestCaseFinished = true;
break; break;
} }
default:
break;
} }
return; return code;
} }
int32_t qwtCreateExecTask(void *tsdb, int32_t vgId, uint64_t taskId, struct SSubplan *pPlan, qTaskInfo_t *pTaskInfo, int32_t qwtCreateExecTask(void *tsdb, int32_t vgId, uint64_t taskId, struct SSubplan *pPlan, qTaskInfo_t *pTaskInfo,
@ -292,6 +331,9 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock **pRes, uint64_t *useconds) {
if (endExec) { if (endExec) {
*pRes = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); *pRes = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
if (NULL == *pRes) {
return terrno;
}
(*pRes)->info.rows = taosRand() % 1000 + 1; (*pRes)->info.rows = taosRand() % 1000 + 1;
} else { } else {
*pRes = NULL; *pRes = NULL;
@ -631,7 +673,7 @@ void *queryThread(void *param) {
while (!qwtTestStop) { while (!qwtTestStop) {
qwtBuildQueryReqMsg(&queryRpc); qwtBuildQueryReqMsg(&queryRpc);
qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc, 0); (void)qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc, 0); // ignore error
if (qwtTestEnableSleep) { if (qwtTestEnableSleep) {
taosUsleep(taosRand() % 5); taosUsleep(taosRand() % 5);
} }
@ -653,7 +695,7 @@ void *fetchThread(void *param) {
while (!qwtTestStop) { while (!qwtTestStop) {
qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc);
code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc, 0); (void)qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc, 0); // ignore error
if (qwtTestEnableSleep) { if (qwtTestEnableSleep) {
taosUsleep(taosRand() % 5); taosUsleep(taosRand() % 5);
} }
@ -674,8 +716,11 @@ void *dropThread(void *param) {
STaskDropReq dropMsg = {0}; STaskDropReq dropMsg = {0};
while (!qwtTestStop) { while (!qwtTestStop) {
qwtBuildDropReqMsg(&dropMsg, &dropRpc); if (0 != qwtBuildDropReqMsg(&dropMsg, &dropRpc)) {
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc, 0); break;
}
(void)qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc, 0); // ignore error
if (qwtTestEnableSleep) { if (qwtTestEnableSleep) {
taosUsleep(taosRand() % 5); taosUsleep(taosRand() % 5);
} }
@ -700,7 +745,7 @@ void *qwtclientThread(void *param) {
qwtTestCaseFinished = false; qwtTestCaseFinished = false;
qwtBuildQueryReqMsg(&queryRpc); qwtBuildQueryReqMsg(&queryRpc);
qwtPutReqToQueue((void *)0x1, QUERY_QUEUE, &queryRpc); (void)qwtPutReqToQueue((void *)0x1, QUERY_QUEUE, &queryRpc); //ignore error
while (!qwtTestCaseFinished) { while (!qwtTestCaseFinished) {
taosUsleep(1); taosUsleep(1);
@ -752,9 +797,9 @@ void *queryQueueThread(void *param) {
} }
if (TDMT_SCH_QUERY == queryRpc->msgType) { if (TDMT_SCH_QUERY == queryRpc->msgType) {
qWorkerProcessQueryMsg(mockPointer, mgmt, queryRpc, 0); (void)qWorkerProcessQueryMsg(mockPointer, mgmt, queryRpc, 0); //ignore error
} else if (TDMT_SCH_QUERY_CONTINUE == queryRpc->msgType) { } else if (TDMT_SCH_QUERY_CONTINUE == queryRpc->msgType) {
qWorkerProcessCQueryMsg(mockPointer, mgmt, queryRpc, 0); (void)qWorkerProcessCQueryMsg(mockPointer, mgmt, queryRpc, 0); //ignore error
} else { } else {
printf("unknown msg in query queue, type:%d\n", queryRpc->msgType); printf("unknown msg in query queue, type:%d\n", queryRpc->msgType);
assert(0); assert(0);
@ -810,16 +855,16 @@ void *fetchQueueThread(void *param) {
switch (fetchRpc->msgType) { switch (fetchRpc->msgType) {
case TDMT_SCH_FETCH: case TDMT_SCH_FETCH:
case TDMT_SCH_MERGE_FETCH: case TDMT_SCH_MERGE_FETCH:
qWorkerProcessFetchMsg(mockPointer, mgmt, fetchRpc, 0); (void)qWorkerProcessFetchMsg(mockPointer, mgmt, fetchRpc, 0); //ignore error
break; break;
case TDMT_SCH_CANCEL_TASK: case TDMT_SCH_CANCEL_TASK:
//qWorkerProcessCancelMsg(mockPointer, mgmt, fetchRpc, 0); //qWorkerProcessCancelMsg(mockPointer, mgmt, fetchRpc, 0);
break; break;
case TDMT_SCH_DROP_TASK: case TDMT_SCH_DROP_TASK:
qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc, 0); (void)qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc, 0); //ignore error
break; break;
case TDMT_SCH_TASK_NOTIFY: case TDMT_SCH_TASK_NOTIFY:
qWorkerProcessNotifyMsg(mockPointer, mgmt, fetchRpc, 0); (void)qWorkerProcessNotifyMsg(mockPointer, mgmt, fetchRpc, 0); //ignore error
break; break;
default: default:
printf("unknown msg type:%d in fetch queue", fetchRpc->msgType); printf("unknown msg type:%d in fetch queue", fetchRpc->msgType);
@ -853,7 +898,7 @@ TEST(seqTest, normalCase) {
qwtBuildQueryReqMsg(&queryRpc); qwtBuildQueryReqMsg(&queryRpc);
qwtBuildFetchReqMsg(&qwtfetchMsg, &fetchRpc); qwtBuildFetchReqMsg(&qwtfetchMsg, &fetchRpc);
qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); (void)qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); //ignore error
stubSetStringToPlan(); stubSetStringToPlan();
stubSetRpcSendResponse(); stubSetRpcSendResponse();
@ -898,7 +943,7 @@ TEST(seqTest, cancelFirst) {
qwtInitLogFile(); qwtInitLogFile();
qwtBuildQueryReqMsg(&queryRpc); qwtBuildQueryReqMsg(&queryRpc);
qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); (void)qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); //ignore error
stubSetStringToPlan(); stubSetStringToPlan();
stubSetRpcSendResponse(); stubSetRpcSendResponse();
@ -954,7 +999,7 @@ TEST(seqTest, randCase) {
if (r >= 0 && r < maxr / 5) { if (r >= 0 && r < maxr / 5) {
printf("Query,%d\n", t++); printf("Query,%d\n", t++);
qwtBuildQueryReqMsg(&queryRpc); qwtBuildQueryReqMsg(&queryRpc);
code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc, 0); (void)qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc, 0); //ignore error
} else if (r >= maxr / 5 && r < maxr * 2 / 5) { } else if (r >= maxr / 5 && r < maxr * 2 / 5) {
// printf("Ready,%d\n", t++); // printf("Ready,%d\n", t++);
// qwtBuildReadyReqMsg(&readyMsg, &readyRpc); // qwtBuildReadyReqMsg(&readyMsg, &readyRpc);
@ -965,14 +1010,14 @@ TEST(seqTest, randCase) {
} else if (r >= maxr * 2 / 5 && r < maxr * 3 / 5) { } else if (r >= maxr * 2 / 5 && r < maxr * 3 / 5) {
printf("Fetch,%d\n", t++); printf("Fetch,%d\n", t++);
qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc);
code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc, 0); (void)qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc, 0); //ignore error
if (qwtTestEnableSleep) { if (qwtTestEnableSleep) {
taosUsleep(1); taosUsleep(1);
} }
} else if (r >= maxr * 3 / 5 && r < maxr * 4 / 5) { } else if (r >= maxr * 3 / 5 && r < maxr * 4 / 5) {
printf("Drop,%d\n", t++); printf("Drop,%d\n", t++);
qwtBuildDropReqMsg(&dropMsg, &dropRpc); (void)qwtBuildDropReqMsg(&dropMsg, &dropRpc); //ignore error
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc, 0); (void)qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc, 0); //ignore error
if (qwtTestEnableSleep) { if (qwtTestEnableSleep) {
taosUsleep(1); taosUsleep(1);
} }
@ -1018,14 +1063,14 @@ TEST(seqTest, multithreadRand) {
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); (void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5, t6; TdThread t1, t2, t3, t4, t5, t6;
taosThreadCreate(&(t1), &thattr, queryThread, mgmt); (void)taosThreadCreate(&(t1), &thattr, queryThread, mgmt); //ignore error
// taosThreadCreate(&(t2), &thattr, readyThread, NULL); // (void)taosThreadCreate(&(t2), &thattr, readyThread, NULL); //ignore error
taosThreadCreate(&(t3), &thattr, fetchThread, NULL); (void)taosThreadCreate(&(t3), &thattr, fetchThread, NULL); //ignore error
taosThreadCreate(&(t4), &thattr, dropThread, NULL); (void)taosThreadCreate(&(t4), &thattr, dropThread, NULL); //ignore error
taosThreadCreate(&(t6), &thattr, fetchQueueThread, mgmt); (void)taosThreadCreate(&(t6), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) { while (true) {
if (qwtTestDeadLoop) { if (qwtTestDeadLoop) {
@ -1083,16 +1128,16 @@ TEST(rcTest, shortExecshortDelay) {
qwtTestMaxExecTaskUsec = 0; qwtTestMaxExecTaskUsec = 0;
qwtTestReqMaxDelayUsec = 0; qwtTestReqMaxDelayUsec = 0;
tsem_init(&qwtTestQuerySem, 0, 0); (void)tsem_init(&qwtTestQuerySem, 0, 0); //ignore error
tsem_init(&qwtTestFetchSem, 0, 0); (void)tsem_init(&qwtTestFetchSem, 0, 0); //ignore error
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); (void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5; TdThread t1, t2, t3, t4, t5;
taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); (void)taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); //ignore error
taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); (void)taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); //ignore error
taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); (void)taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) { while (true) {
if (qwtTestDeadLoop) { if (qwtTestDeadLoop) {
@ -1114,8 +1159,8 @@ TEST(rcTest, shortExecshortDelay) {
if (qwtTestCaseFinished) { if (qwtTestCaseFinished) {
if (qwtTestQuitThreadNum < 3) { if (qwtTestQuitThreadNum < 3) {
tsem_post(&qwtTestQuerySem); (void)tsem_post(&qwtTestQuerySem); //ignore error
tsem_post(&qwtTestFetchSem); (void)tsem_post(&qwtTestFetchSem); //ignore error
taosUsleep(10); taosUsleep(10);
} }
@ -1166,16 +1211,16 @@ TEST(rcTest, longExecshortDelay) {
qwtTestMaxExecTaskUsec = 1000000; qwtTestMaxExecTaskUsec = 1000000;
qwtTestReqMaxDelayUsec = 0; qwtTestReqMaxDelayUsec = 0;
tsem_init(&qwtTestQuerySem, 0, 0); (void)tsem_init(&qwtTestQuerySem, 0, 0); //ignore error
tsem_init(&qwtTestFetchSem, 0, 0); (void)tsem_init(&qwtTestFetchSem, 0, 0); //ignore error
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); (void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5; TdThread t1, t2, t3, t4, t5;
taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); (void)taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); //ignore error
taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); (void)taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); //ignore error
taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); (void)taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) { while (true) {
if (qwtTestDeadLoop) { if (qwtTestDeadLoop) {
@ -1197,8 +1242,8 @@ TEST(rcTest, longExecshortDelay) {
if (qwtTestCaseFinished) { if (qwtTestCaseFinished) {
if (qwtTestQuitThreadNum < 3) { if (qwtTestQuitThreadNum < 3) {
tsem_post(&qwtTestQuerySem); (void)tsem_post(&qwtTestQuerySem); //ignore error
tsem_post(&qwtTestFetchSem); (void)tsem_post(&qwtTestFetchSem); //ignore error
taosUsleep(10); taosUsleep(10);
} }
@ -1249,16 +1294,16 @@ TEST(rcTest, shortExeclongDelay) {
qwtTestMaxExecTaskUsec = 0; qwtTestMaxExecTaskUsec = 0;
qwtTestReqMaxDelayUsec = 1000000; qwtTestReqMaxDelayUsec = 1000000;
tsem_init(&qwtTestQuerySem, 0, 0); (void)tsem_init(&qwtTestQuerySem, 0, 0); //ignore error
tsem_init(&qwtTestFetchSem, 0, 0); (void)tsem_init(&qwtTestFetchSem, 0, 0); //ignore error
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); (void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5; TdThread t1, t2, t3, t4, t5;
taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); (void)taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); //ignore error
taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); (void)taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); //ignore error
taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); (void)taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) { while (true) {
if (qwtTestDeadLoop) { if (qwtTestDeadLoop) {
@ -1280,8 +1325,8 @@ TEST(rcTest, shortExeclongDelay) {
if (qwtTestCaseFinished) { if (qwtTestCaseFinished) {
if (qwtTestQuitThreadNum < 3) { if (qwtTestQuitThreadNum < 3) {
tsem_post(&qwtTestQuerySem); (void)tsem_post(&qwtTestQuerySem); //ignore error
tsem_post(&qwtTestFetchSem); (void)tsem_post(&qwtTestFetchSem); //ignore error
taosUsleep(10); taosUsleep(10);
} }
@ -1327,16 +1372,16 @@ TEST(rcTest, dropTest) {
code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb); code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
tsem_init(&qwtTestQuerySem, 0, 0); (void)tsem_init(&qwtTestQuerySem, 0, 0); //ignore error
tsem_init(&qwtTestFetchSem, 0, 0); (void)tsem_init(&qwtTestFetchSem, 0, 0); //ignore error
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); (void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5; TdThread t1, t2, t3, t4, t5;
taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); (void)taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); //ignore error
taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); (void)taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); //ignore error
taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); (void)taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) { while (true) {
if (qwtTestDeadLoop) { if (qwtTestDeadLoop) {

View File

@ -1654,6 +1654,12 @@ static int32_t sclGetCompOperatorResType(SOperatorNode *pOp) {
(!IS_STR_DATA_TYPE(rdt.type) && (rdt.type != TSDB_DATA_TYPE_NULL))) { (!IS_STR_DATA_TYPE(rdt.type) && (rdt.type != TSDB_DATA_TYPE_NULL))) {
return TSDB_CODE_TSC_INVALID_OPERATION; return TSDB_CODE_TSC_INVALID_OPERATION;
} }
if (nodesIsMatchRegularOp(pOp)) {
SValueNode* node = (SValueNode*)(pOp->pRight);
if(checkRegexPattern(node->literal) != TSDB_CODE_SUCCESS){
return TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR;
}
}
} }
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;

View File

@ -1675,10 +1675,8 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
pRes[i] = false; pRes[i] = false;
continue; continue;
} }
char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); char *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
char *pRightData = colDataGetData(pRight->columnData, rightIndex); char *pRightData = colDataGetData(pRight->columnData, rightIndex);
pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData);
if (pRes[i]) { if (pRes[i]) {
++num; ++num;
@ -1714,7 +1712,6 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
if (!pLeftData || !pRightData) { if (!pLeftData || !pRightData) {
result = false; result = false;
} }
if (!result) { if (!result) {
colDataSetInt8(pOut->columnData, i, (int8_t *)&result); colDataSetInt8(pOut->columnData, i, (int8_t *)&result);
} else { } else {

View File

@ -43,6 +43,7 @@
#include "tglobal.h" #include "tglobal.h"
#include "tlog.h" #include "tlog.h"
#include "tvariant.h" #include "tvariant.h"
#include "tcompare.h"
#define _DEBUG_PRINT_ 0 #define _DEBUG_PRINT_ 0
@ -52,6 +53,12 @@
#define PRINTF(...) #define PRINTF(...)
#endif #endif
class constantTest {
public:
constantTest() { InitRegexCache(); }
~constantTest() { DestroyRegexCache(); }
};
static constantTest test;
namespace { namespace {
SColumnInfo createColumnInfo(int32_t colId, int32_t type, int32_t bytes) { SColumnInfo createColumnInfo(int32_t colId, int32_t type, int32_t bytes) {

View File

@ -583,7 +583,7 @@ char *schGetOpStr(SCH_OP_TYPE type);
int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync);
int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq);
int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq); int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq);
int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes); void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes);
int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet); int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet);
int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode); int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode);
void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode); void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode);
@ -606,7 +606,7 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask);
void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); void schDropTaskInHashList(SSchJob *pJob, SHashObj *list);
int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask); int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask);
int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level);
int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask);
int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel);
int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask);
void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode); void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode);

View File

@ -16,17 +16,5 @@
#include "query.h" #include "query.h"
#include "schInt.h" #include "schInt.h"
tsem_t schdRspSem;
SSchDebug gSCHDebug = {0}; SSchDebug gSCHDebug = {0};
void schdExecCallback(SExecResult* pResult, void* param, int32_t code) {
if (code) {
pResult->code = code;
}
*(SExecResult*)param = *pResult;
taosMemoryFree(pResult);
tsem_post(&schdRspSem);
}

View File

@ -50,6 +50,10 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks); int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks);
for (int32_t i = 0; i < taskNum; ++i) { for (int32_t i = 0; i < taskNum; ++i) {
SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i); SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i);
if (NULL == pTask) {
SCH_JOB_DLOG("fail to get the %dth task", i);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
sum += pTask->plan->execNodeStat.tableNum; sum += pTask->plan->execNodeStat.tableNum;
} }
@ -214,6 +218,10 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
for (int32_t i = 0; i < taskNum; ++i) { for (int32_t i = 0; i < taskNum; ++i) {
pTask = *(SSchTask **)taosArrayGet(ctrl->taskList, i); pTask = *(SSchTask **)taosArrayGet(ctrl->taskList, i);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task", i);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) { if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) {
@ -243,6 +251,11 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
if (i < (taskNum - 1)) { if (i < (taskNum - 1)) {
SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList); SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList);
if (NULL == pLastTask) {
SCH_JOB_ELOG("fail to get the last task, num:%d", (int32_t)taosArrayGetSize(ctrl->taskList));
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (remainNum < pLastTask->plan->execNodeStat.tableNum) { if (remainNum < pLastTask->plan->execNodeStat.tableNum) {
SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d, smallestInList:%d", SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d, smallestInList:%d",
ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum); ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum);

View File

@ -47,6 +47,7 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) {
return; return;
_return: _return:
SCH_JOB_DLOG("job errCode updated to %s", tstrerror(errCode)); SCH_JOB_DLOG("job errCode updated to %s", tstrerror(errCode));
} }
@ -166,9 +167,18 @@ _return:
int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t i = 0; i < pJob->levelNum; ++i) { for (int32_t i = 0; i < pJob->levelNum; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i); SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, pJob->levelNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
for (int32_t m = 0; m < pLevel->taskNum; ++m) { for (int32_t m = 0; m < pLevel->taskNum; ++m) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); SSchTask *pTask = taosArrayGet(pLevel->subTasks, m);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum: %d", m, pLevel->level, pLevel->taskNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSubplan *pPlan = pTask->plan; SSubplan *pPlan = pTask->plan;
int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0; int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0;
int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0; int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0;
@ -188,6 +198,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t n = 0; n < childNum; ++n) { for (int32_t n = 0; n < childNum; ++n) {
SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n);
if (NULL == child) {
SCH_JOB_ELOG("fail to get the %dth child subplan, childNum: %d", n, childNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES);
if (NULL == childTask || NULL == *childTask) { if (NULL == childTask || NULL == *childTask) {
SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
@ -222,6 +237,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t n = 0; n < parentNum; ++n) { for (int32_t n = 0; n < parentNum; ++n) {
SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n);
if (NULL == parent) {
SCH_JOB_ELOG("fail to get the %dth parent subplan, parentNum: %d", n, parentNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES);
if (NULL == parentTask || NULL == *parentTask) { if (NULL == parentTask || NULL == *parentTask) {
SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
@ -241,6 +261,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
} }
SSchLevel *pLevel = taosArrayGet(pJob->levels, 0); SSchLevel *pLevel = taosArrayGet(pJob->levels, 0);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get level 0 level, levelNum:%d", (int32_t)taosArrayGetSize(pJob->levels));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (SCH_IS_QUERY_JOB(pJob)) { if (SCH_IS_QUERY_JOB(pJob)) {
if (pLevel->taskNum > 1) { if (pLevel->taskNum > 1) {
SCH_JOB_ELOG("invalid query plan, level:0, taskNum:%d", pLevel->taskNum); SCH_JOB_ELOG("invalid query plan, level:0, taskNum:%d", pLevel->taskNum);
@ -248,6 +273,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
} }
SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0); SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the first task in level 0, taskNum:%d", (int32_t)taosArrayGetSize(pLevel->subTasks));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType || EXPLAIN_MODE_DISABLE != pJob->attr.explainMode) { if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType || EXPLAIN_MODE_DISABLE != pJob->attr.explainMode) {
pJob->attr.needFetch = true; pJob->attr.needFetch = true;
} }
@ -261,7 +291,9 @@ int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
taosArrayPush(pJob->dataSrcTasks, &pTask); if (NULL == taosArrayPush(pJob->dataSrcTasks, &pTask)) {
SCH_ERR_RET(terrno);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -318,6 +350,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
} }
pLevel = taosArrayGet(pJob->levels, i); pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, levelNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
pLevel->level = i; pLevel->level = i;
plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i);
@ -342,6 +379,10 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
for (int32_t n = 0; n < taskNum; ++n) { for (int32_t n = 0; n < taskNum; ++n) {
SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n);
if (NULL == plan) {
SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d", n, taskNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_SET_JOB_TYPE(pJob, plan->subplanType); SCH_SET_JOB_TYPE(pJob, plan->subplanType);
@ -383,7 +424,7 @@ _return:
SCH_RET(code); SCH_RET(code);
} }
int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) {
pRes->code = atomic_load_32(&pJob->errCode); pRes->code = atomic_load_32(&pJob->errCode);
pRes->numOfRows = pJob->resNumOfRows; pRes->numOfRows = pJob->resNumOfRows;
@ -395,8 +436,6 @@ int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) {
SCH_UNLOCK(SCH_WRITE, &pJob->resLock); SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
SCH_JOB_DLOG("execRes dumped, code: %s", tstrerror(pRes->code)); SCH_JOB_DLOG("execRes dumped, code: %s", tstrerror(pRes->code));
return TSDB_CODE_SUCCESS;
} }
int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) {
@ -421,10 +460,13 @@ int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) {
if (NULL == *pData) { if (NULL == *pData) {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp)); SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp));
if (rsp) { if (NULL == rsp) {
rsp->completed = 1; SCH_JOB_ELOG("malloc SRetrieveTableRsp %d failed, code:%x", (int32_t)sizeof(SRetrieveTableRsp), terrno);
SCH_ERR_JRET(terrno);
} }
rsp->completed = 1;
*pData = rsp; *pData = rsp;
SCH_JOB_DLOG("empty res and set query complete, code:%x", code); SCH_JOB_DLOG("empty res and set query complete, code:%x", code);
} }
@ -440,10 +482,13 @@ _return:
int32_t schNotifyUserExecRes(SSchJob *pJob) { int32_t schNotifyUserExecRes(SSchJob *pJob) {
SExecResult *pRes = taosMemoryCalloc(1, sizeof(SExecResult)); SExecResult *pRes = taosMemoryCalloc(1, sizeof(SExecResult));
if (pRes) { if (NULL == pRes) {
schDumpJobExecRes(pJob, pRes); qError("malloc execResult %d failed, error: %x", (int32_t)sizeof(SExecResult), terrno);
SCH_RET(terrno);
} }
schDumpJobExecRes(pJob, pRes);
SCH_JOB_DLOG("sch start to invoke exec cb, code: %s", tstrerror(pJob->errCode)); SCH_JOB_DLOG("sch start to invoke exec cb, code: %s", tstrerror(pJob->errCode));
(*pJob->userRes.execFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode)); (*pJob->userRes.execFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode));
SCH_JOB_DLOG("sch end from exec cb, code: %s", tstrerror(pJob->errCode)); SCH_JOB_DLOG("sch end from exec cb, code: %s", tstrerror(pJob->errCode));
@ -454,7 +499,10 @@ int32_t schNotifyUserExecRes(SSchJob *pJob) {
int32_t schNotifyUserFetchRes(SSchJob *pJob) { int32_t schNotifyUserFetchRes(SSchJob *pJob) {
void *pRes = NULL; void *pRes = NULL;
schDumpJobFetchRes(pJob, &pRes); int32_t code = schDumpJobFetchRes(pJob, &pRes);
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_SUCCESS == atomic_load_32(&pJob->errCode)) {
atomic_store_32(&pJob->errCode, code);
}
SCH_JOB_DLOG("sch start to invoke fetch cb, code: %s", tstrerror(pJob->errCode)); SCH_JOB_DLOG("sch start to invoke fetch cb, code: %s", tstrerror(pJob->errCode));
(*pJob->userRes.fetchFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode)); (*pJob->userRes.fetchFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode));
@ -478,13 +526,13 @@ void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) {
if (SCH_JOB_IN_SYNC_OP(pJob)) { if (SCH_JOB_IN_SYNC_OP(pJob)) {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
tsem_post(&pJob->rspSem); (void)tsem_post(&pJob->rspSem); // ignore error
} else if (SCH_JOB_IN_ASYNC_EXEC_OP(pJob)) { } else if (SCH_JOB_IN_ASYNC_EXEC_OP(pJob)) {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
schNotifyUserExecRes(pJob); (void)schNotifyUserExecRes(pJob); // ignore error
} else if (SCH_JOB_IN_ASYNC_FETCH_OP(pJob)) { } else if (SCH_JOB_IN_ASYNC_FETCH_OP(pJob)) {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
schNotifyUserFetchRes(pJob); (void)schNotifyUserFetchRes(pJob); // ignore error
} else { } else {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
SCH_JOB_ELOG("job not in any operation, status:%s", jobTaskStatusStr(pJob->status)); SCH_JOB_ELOG("job not in any operation, status:%s", jobTaskStatusStr(pJob->status));
@ -519,7 +567,8 @@ int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode) {
return TSDB_CODE_SCH_IGNORE_ERROR; return TSDB_CODE_SCH_IGNORE_ERROR;
} }
schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); // ignore error
return TSDB_CODE_SCH_IGNORE_ERROR; return TSDB_CODE_SCH_IGNORE_ERROR;
} }
@ -530,7 +579,8 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) {
return TSDB_CODE_SCH_IGNORE_ERROR; return TSDB_CODE_SCH_IGNORE_ERROR;
} }
schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); (void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); // ignore error
return TSDB_CODE_SCH_IGNORE_ERROR; return TSDB_CODE_SCH_IGNORE_ERROR;
} }
@ -569,12 +619,21 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) {
SSchLevel *pLevel = pTask->level; SSchLevel *pLevel = pTask->level;
int32_t doneNum = atomic_load_32(&pLevel->taskExecDoneNum); int32_t doneNum = atomic_load_32(&pLevel->taskExecDoneNum);
if (doneNum == pLevel->taskNum) { if (doneNum == pLevel->taskNum) {
atomic_sub_fetch_32(&pJob->levelIdx, 1); (void)atomic_sub_fetch_32(&pJob->levelIdx, 1);
pLevel = taosArrayGet(pJob->levels, pJob->levelIdx); pLevel = taosArrayGet(pJob->levels, pJob->levelIdx);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
for (int32_t i = 0; i < pLevel->taskNum; ++i) { for (int32_t i = 0; i < pLevel->taskNum; ++i) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, i); SSchTask *pTask = taosArrayGet(pLevel->subTasks, i);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", i, pLevel->level, pLevel->taskNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (pTask->children && taosArrayGetSize(pTask->children) > 0) { if (pTask->children && taosArrayGetSize(pTask->children) > 0) {
continue; continue;
} }
@ -602,7 +661,11 @@ int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) {
} }
} }
taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo)); if (NULL == taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo))) {
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
SCH_ERR_RET(terrno);
}
taosArrayDestroy(rsp->tbVerInfo); taosArrayDestroy(rsp->tbVerInfo);
pJob->execRes.msgType = TDMT_SCH_QUERY; pJob->execRes.msgType = TDMT_SCH_QUERY;
@ -629,6 +692,11 @@ int32_t schLaunchJob(SSchJob *pJob) {
SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL)); SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL));
} else { } else {
SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx);
if (NULL == level) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_ERR_RET(schLaunchLevelTasks(pJob, level)); SCH_ERR_RET(schLaunchLevelTasks(pJob, level));
} }
@ -661,10 +729,19 @@ void schFreeJobImpl(void *job) {
int32_t numOfLevels = taosArrayGetSize(pJob->levels); int32_t numOfLevels = taosArrayGetSize(pJob->levels);
for (int32_t i = 0; i < numOfLevels; ++i) { for (int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i); SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels);
continue;
}
int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
for (int32_t j = 0; j < numOfTasks; ++j) { for (int32_t j = 0; j < numOfTasks; ++j) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth task, taskNum:%d", j, numOfTasks);
continue;
}
schFreeTask(pJob, pTask); schFreeTask(pJob, pTask);
} }
@ -687,12 +764,12 @@ void schFreeJobImpl(void *job) {
destroyQueryExecRes(&pJob->execRes); destroyQueryExecRes(&pJob->execRes);
qDestroyQueryPlan(pJob->pDag); qDestroyQueryPlan(pJob->pDag);
nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); (void)nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); // ignore error
taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->userRes.execRes);
taosMemoryFreeClear(pJob->fetchRes); taosMemoryFreeClear(pJob->fetchRes);
taosMemoryFreeClear(pJob->sql); taosMemoryFreeClear(pJob->sql);
tsem_destroy(&pJob->rspSem); (void)tsem_destroy(&pJob->rspSem); // ignore error
taosMemoryFree(pJob); taosMemoryFree(pJob);
int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1);
@ -711,7 +788,7 @@ int32_t schJobFetchRows(SSchJob *pJob) {
if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) {
SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
tsem_wait(&pJob->rspSem); (void)tsem_wait(&pJob->rspSem); // ignore error
SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes)); SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes));
} }
} else { } else {
@ -739,9 +816,19 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
pJob->conn = *pReq->pConn; pJob->conn = *pReq->pConn;
if (pReq->sql) { if (pReq->sql) {
pJob->sql = taosStrdup(pReq->sql); pJob->sql = taosStrdup(pReq->sql);
if (NULL == pJob->sql) {
qError("QID:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql);
SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
pJob->pDag = pReq->pDag; pJob->pDag = pReq->pDag;
pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); if (pReq->allocatorRefId > 0) {
pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId);
if (pJob->allocatorRefId <= 0) {
qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId);
SCH_ERR_JRET(terrno);
}
}
pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillFp = pReq->chkKillFp;
pJob->chkKillParam = pReq->chkKillParam; pJob->chkKillParam = pReq->chkKillParam;
pJob->userRes.execFp = pReq->execFp; pJob->userRes.execFp = pReq->execFp;
@ -752,6 +839,10 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId);
} else { } else {
pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL); pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL);
if (NULL == pJob->nodeList) {
qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, (int32_t)taosArrayGetSize(pReq->pNodeList));
SCH_ERR_JRET(terrno);
}
} }
pJob->taskList = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, pJob->taskList = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false,
@ -785,7 +876,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
SCH_ERR_JRET(terrno); SCH_ERR_JRET(terrno);
} }
atomic_add_fetch_32(&schMgmt.jobNum, 1); (void)atomic_add_fetch_32(&schMgmt.jobNum, 1);
*pJobId = pJob->refId; *pJobId = pJob->refId;
@ -800,7 +891,7 @@ _return:
} else if (pJob->refId < 0) { } else if (pJob->refId < 0) {
schFreeJobImpl(pJob); schFreeJobImpl(pJob);
} else { } else {
taosRemoveRef(schMgmt.jobRef, pJob->refId); (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error
} }
SCH_RET(code); SCH_RET(code);
@ -814,7 +905,7 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) {
if (pReq->syncReq) { if (pReq->syncReq) {
SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
tsem_wait(&pJob->rspSem); (void)tsem_wait(&pJob->rspSem); // ignore error
} }
SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId);
@ -845,7 +936,7 @@ int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) {
} }
SCH_UNLOCK(SCH_WRITE, &pJob->resLock); SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); SCH_ERR_RET(schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC));
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -866,6 +957,10 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) {
int32_t numOfLevels = taosArrayGetSize(pJob->levels); int32_t numOfLevels = taosArrayGetSize(pJob->levels);
for (int32_t i = 0; i < numOfLevels; ++i) { for (int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i); SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
pLevel->taskExecDoneNum = 0; pLevel->taskExecDoneNum = 0;
pLevel->taskLaunchedNum = 0; pLevel->taskLaunchedNum = 0;
@ -873,6 +968,11 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) {
int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
for (int32_t j = 0; j < numOfTasks; ++j) { for (int32_t j = 0; j < numOfTasks; ++j) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", j, i, numOfTasks);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_LOCK_TASK(pTask); SCH_LOCK_TASK(pTask);
code = schChkUpdateRedirectCtx(pJob, pTask, NULL, rspCode); code = schChkUpdateRedirectCtx(pJob, pTask, NULL, rspCode);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
@ -973,7 +1073,7 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int
} }
if (errCode) { if (errCode) {
schHandleJobFailure(pJob, errCode); (void)schHandleJobFailure(pJob, errCode); // handle internal
} }
SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode)); SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode));
@ -1052,11 +1152,11 @@ void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode) {
} }
if (errCode) { if (errCode) {
schHandleJobFailure(pJob, errCode); (void)schHandleJobFailure(pJob, errCode); // ignore error
} }
if (pJob) { if (pJob) {
schReleaseJob(pJob->refId); (void)schReleaseJob(pJob->refId); // ignore error
} }
} }
@ -1091,7 +1191,7 @@ _return:
SCH_UNLOCK_TASK(pTask); SCH_UNLOCK_TASK(pTask);
} }
if (pJob) { if (pJob) {
schReleaseJob(rId); (void)schReleaseJob(rId); // ignore error
} }
SCH_RET(code); SCH_RET(code);

View File

@ -316,18 +316,18 @@ void schFreeRpcCtx(SRpcCtx *pCtx) {
} }
} }
int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) {
*pTask = NULL;
int32_t s = taosHashGetSize(pTaskList); int32_t s = taosHashGetSize(pTaskList);
if (s <= 0) { if (s <= 0) {
return TSDB_CODE_SUCCESS; return;
} }
SSchTask **task = taosHashGet(pTaskList, &taskId, sizeof(taskId)); SSchTask **task = taosHashGet(pTaskList, &taskId, sizeof(taskId));
if (NULL == task || NULL == (*task)) { if (NULL == task || NULL == (*task)) {
return TSDB_CODE_SUCCESS; return;
} }
*pTask = *task; *pTask = *task;
return TSDB_CODE_SUCCESS;
} }

View File

@ -108,14 +108,26 @@ int32_t schedulerGetTasksStatus(int64_t jobId, SArray *pSub) {
for (int32_t i = pJob->levelNum - 1; i >= 0; --i) { for (int32_t i = pJob->levelNum - 1; i >= 0; --i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i); SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
qError("failed to get level %d", i);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
for (int32_t m = 0; m < pLevel->taskNum; ++m) { for (int32_t m = 0; m < pLevel->taskNum; ++m) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); SSchTask *pTask = taosArrayGet(pLevel->subTasks, m);
if (NULL == pTask) {
qError("failed to get task %d, total: %d", m, pLevel->taskNum);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SQuerySubDesc subDesc = {0}; SQuerySubDesc subDesc = {0};
subDesc.tid = pTask->taskId; subDesc.tid = pTask->taskId;
strcpy(subDesc.status, jobTaskStatusStr(pTask->status)); TAOS_STRCPY(subDesc.status, jobTaskStatusStr(pTask->status));
taosArrayPush(pSub, &subDesc); if (NULL == taosArrayPush(pSub, &subDesc)) {
qError("taosArrayPush task %d failed, error: %x, ", m, terrno);
SCH_ERR_JRET(terrno);
}
} }
} }
@ -141,7 +153,7 @@ int32_t schedulerUpdatePolicy(int32_t policy) {
qDebug("schedule policy updated to %d", schMgmt.cfg.schPolicy); qDebug("schedule policy updated to %d", schMgmt.cfg.schPolicy);
break; break;
default: default:
return TSDB_CODE_TSC_INVALID_INPUT; SCH_RET(TSDB_CODE_TSC_INVALID_INPUT);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -159,14 +171,14 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) {
SSchJob *pJob = schAcquireJob(*jobId); SSchJob *pJob = schAcquireJob(*jobId);
if (NULL == pJob) { if (NULL == pJob) {
qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId);
return; return;
} }
SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode)); SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode));
schHandleJobDrop(pJob, errCode); (void)schHandleJobDrop(pJob, errCode); // ignore any error
schReleaseJob(*jobId); (void)schReleaseJob(*jobId); // ignore error
*jobId = 0; *jobId = 0;
} }
@ -182,7 +194,7 @@ void schedulerDestroy(void) {
if (refId == 0) { if (refId == 0) {
break; break;
} }
taosRemoveRef(schMgmt.jobRef, pJob->refId); (void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error
pJob = taosIterateRef(schMgmt.jobRef, refId); pJob = taosIterateRef(schMgmt.jobRef, refId);
} }

View File

@ -308,7 +308,9 @@ int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) {
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
ASSERT(pInfo->inCheckProcess == 0); ASSERT(pInfo->inCheckProcess == 0);
pInfo->pList = taosArrayDestroy(pInfo->pList); taosArrayDestroy(pInfo->pList);
pInfo->pList = NULL;
if (pInfo->checkRspTmr != NULL) { if (pInfo->checkRspTmr != NULL) {
/*bool ret = */ taosTmrStop(pInfo->checkRspTmr); /*bool ret = */ taosTmrStop(pInfo->checkRspTmr);
pInfo->checkRspTmr = NULL; pInfo->checkRspTmr = NULL;

View File

@ -438,11 +438,13 @@ void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) {
} }
if (pMsg->pUpdateNodes != NULL) { if (pMsg->pUpdateNodes != NULL) {
pMsg->pUpdateNodes = taosArrayDestroy(pMsg->pUpdateNodes); taosArrayDestroy(pMsg->pUpdateNodes);
pMsg->pUpdateNodes = NULL;
} }
if (pMsg->pTaskStatus != NULL) { if (pMsg->pTaskStatus != NULL) {
pMsg->pTaskStatus = taosArrayDestroy(pMsg->pTaskStatus); taosArrayDestroy(pMsg->pTaskStatus);
pMsg->pTaskStatus = NULL;
} }
pMsg->msgId = -1; pMsg->msgId = -1;

Some files were not shown because too many files have changed in this diff Show More