xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/3_liaohj

This commit is contained in:
Haojun Liao 2024-07-17 11:25:19 +08:00
parent 9aadc5e4ac 23e72a6e69
commit 53ed030967
128 changed files with 9753 additions and 7295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/common/cos_cp.h
View File

@ -60,8 +60,8 @@ typedef struct {
} SCheckpoint;
int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint);
void cos_cp_close(TdFilePtr fd);
void cos_cp_remove(char const* filepath);
int32_t cos_cp_close(TdFilePtr fd);
int32_t cos_cp_remove(char const* filepath);
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
int32_t cos_cp_dump(SCheckpoint* checkpoint);

2
include/common/tdataformat.h
View File

@ -136,7 +136,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
void tRowGetPrimaryKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2);
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
// SRowIter ================================
int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);

2
include/common/tmisce.h
View File

@ -31,7 +31,7 @@ typedef struct SCorEpSet {
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t len);
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
void epsetAssign(SEpSet* dst, const SEpSet* pSrc);

4
include/common/tmsgcb.h
View File

@ -17,13 +17,13 @@
#define _TD_COMMON_MSG_CB_H_
#include "os.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SRpcMsg SRpcMsg;
typedef struct SEpSet SEpSet;
typedef struct SMgmtWrapper SMgmtWrapper;
typedef struct SRpcHandleInfo SRpcHandleInfo;
@ -46,7 +46,7 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg);
typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
typedef void (*SendRspFp)(SRpcMsg* pMsg);
typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg);
typedef void (*RegisterBrokenLinkArgFp)(struct SRpcMsg* pMsg);
typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
typedef void (*ReportStartup)(const char* name, const char* desc);

2
include/common/tname.h
View File

@ -81,7 +81,7 @@ typedef struct {
// uint64_t uid; // child table uid, may be useful
} RandTableName;
void buildChildTableName(RandTableName* rName);
int32_t buildChildTableName(RandTableName* rName);
#ifdef __cplusplus
}

20
include/common/ttokendef.h
View File

@ -132,16 +132,16 @@
#define TK_TABLE 114
#define TK_NK_LP 115
#define TK_NK_RP 116
#define TK_STABLE 117
#define TK_COLUMN 118
#define TK_MODIFY 119
#define TK_RENAME 120
#define TK_TAG 121
#define TK_SET 122
#define TK_NK_EQ 123
#define TK_USING 124
#define TK_TAGS 125
#define TK_FILE 126
#define TK_USING 117
#define TK_FILE 118
#define TK_STABLE 119
#define TK_COLUMN 120
#define TK_MODIFY 121
#define TK_RENAME 122
#define TK_TAG 123
#define TK_SET 124
#define TK_NK_EQ 125
#define TK_TAGS 126
#define TK_BOOL 127
#define TK_TINYINT 128
#define TK_SMALLINT 129

2
include/libs/audit/audit.h
View File

@ -50,7 +50,7 @@ typedef struct {
int32_t auditInit(const SAuditCfg *pCfg);
void auditCleanup();
void auditSend(SJson *pJson);
int32_t auditSend(SJson *pJson);
void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,
char *detail, int32_t len);
void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,

2
include/libs/executor/executor.h
View File

@ -152,7 +152,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
* @return
*/
int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
int32_t* tversion, int32_t idx);
int32_t* tversion, int32_t idx, bool* tbGet);
/**
* The main task execution function, including query on both table and multiple tables,

14
include/libs/executor/storageapi.h
View File

@ -105,7 +105,7 @@ typedef struct SMTbCursor {
} SMTbCursor;
typedef struct SMCtbCursor {
SMeta* pMeta;
struct SMeta* pMeta;
void* pCur;
tb_uid_t suid;
void* pKey;
@ -134,7 +134,7 @@ typedef struct SMetaTableInfo {
} SMetaTableInfo;
typedef struct SSnapContext {
SMeta* pMeta;
struct SMeta* pMeta;
int64_t snapVersion;
void* pCur;
int64_t suid;
@ -178,7 +178,7 @@ typedef struct TsdReader {
int32_t (*tsdNextDataBlock)();
int32_t (*tsdReaderRetrieveBlockSMAInfo)();
SSDataBlock *(*tsdReaderRetrieveDataBlock)();
int32_t (*tsdReaderRetrieveDataBlock)();
void (*tsdReaderReleaseDataBlock)();
@ -381,7 +381,7 @@ typedef struct SStateStore {
void** ppVal, int32_t* pVLen);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
@ -389,12 +389,12 @@ typedef struct SStateStore {
void (*updateInfoDestroy)(SUpdateInfo* pInfo);
void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);

3
include/libs/nodes/cmdnodes.h
View File

@ -221,9 +221,6 @@ typedef struct SCreateSubTableFromFileClause {
bool ignoreExists;
SNodeList* pSpecificTags;
char filePath[PATH_MAX];
TdFilePtr fp;
SArray* aCreateTbData;
SArray* aTagIndexs;
} SCreateSubTableFromFileClause;
typedef struct SCreateMultiTablesStmt {

23
include/libs/nodes/querynodes.h
View File

@ -233,7 +233,7 @@ typedef struct SViewNode {
#define IS_WINDOW_JOIN(_stype) ((_stype) == JOIN_STYPE_WIN)
#define IS_ASOF_JOIN(_stype) ((_stype) == JOIN_STYPE_ASOF)
typedef enum EJoinType {
typedef enum EJoinType {
JOIN_TYPE_INNER = 0,
JOIN_TYPE_LEFT,
JOIN_TYPE_RIGHT,
@ -251,7 +251,7 @@ typedef enum EJoinSubType {
JOIN_STYPE_MAX_VALUE
} EJoinSubType;
typedef enum EJoinAlgorithm {
typedef enum EJoinAlgorithm {
JOIN_ALGO_UNKNOWN = 0,
JOIN_ALGO_MERGE,
JOIN_ALGO_HASH,
@ -454,7 +454,7 @@ typedef struct SSetOperator {
SNode* pLimit;
char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision;
ETimeLineMode timeLineResMode;
ETimeLineMode timeLineResMode;
bool timeLineFromOrderBy;
bool joinContains;
} SSetOperator;
@ -504,6 +504,10 @@ typedef void (*FFreeTableBlockHash)(SHashObj*);
typedef void (*FFreeVgourpBlockArray)(SArray*);
struct SStbRowsDataContext;
typedef void (*FFreeStbRowsDataContext)(struct SStbRowsDataContext*);
struct SCreateTbInfo;
struct SParseFileContext;
typedef void (*FDestroyParseFileContext)(struct SParseFileContext**);
typedef struct SVnodeModifyOpStmt {
ENodeType nodeType;
ENodeType sqlNodeType;
@ -524,7 +528,7 @@ typedef struct SVnodeModifyOpStmt {
SHashObj* pTableNameHashObj; // set of table names for refreshing meta, sync mode
SHashObj* pDbFNameHashObj; // set of db names for refreshing meta, sync mode
SHashObj* pTableCxtHashObj; // temp SHashObj<tuid, STableDataCxt*> for single request
SArray* pVgDataBlocks; // SArray<SVgroupDataCxt*>
SArray* pVgDataBlocks; // SArray<SVgroupDataCxt*>
SVCreateTbReq* pCreateTblReq;
TdFilePtr fp;
FFreeTableBlockHash freeHashFunc;
@ -532,9 +536,13 @@ typedef struct SVnodeModifyOpStmt {
bool usingTableProcessing;
bool fileProcessing;
bool stbSyntax;
struct SStbRowsDataContext* pStbRowsCxt;
bool stbSyntax;
struct SStbRowsDataContext* pStbRowsCxt;
FFreeStbRowsDataContext freeStbRowsCxtFunc;
struct SCreateTbInfo* pCreateTbInfo;
struct SParseFileContext* pParFileCxt;
FDestroyParseFileContext destroyParseFileCxt;
} SVnodeModifyOpStmt;
typedef struct SExplainOptions {
@ -602,7 +610,7 @@ typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, C
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols);
int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias, ECollectColType type,
SNodeList** pCols);
SNodeList** pCols);
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols);
typedef bool (*FFuncClassifier)(int32_t funcId);
@ -618,6 +626,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp);
bool nodesIsComparisonOp(const SOperatorNode* pOp);
bool nodesIsJsonOp(const SOperatorNode* pOp);
bool nodesIsRegularOp(const SOperatorNode* pOp);
bool nodesIsMatchRegularOp(const SOperatorNode* pOp);
bool nodesIsBitwiseOp(const SOperatorNode* pOp);
bool nodesExprHasColumn(SNode* pNode);

4
include/libs/parser/parser.h
View File

@ -149,10 +149,10 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
void qDestroyBoundColInfo(void* pInfo);
SQuery* smlInitHandle();
int32_t smlInitHandle(SQuery** query);
int32_t smlBuildRow(STableDataCxt* pTableCxt);
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index);
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta);
int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt);
void clearColValArraySml(SArray* pCols);
int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,

4
include/libs/scheduler/scheduler.h
View File

@ -23,8 +23,6 @@ extern "C" {
#include "catalog.h"
#include "planner.h"
extern tsem_t schdRspSem;
typedef struct SQueryProfileSummary {
int64_t startTs; // Object created and added into the message queue
int64_t endTs; // the timestamp when the task is completed
@ -101,8 +99,6 @@ void schedulerFreeJob(int64_t* job, int32_t errCode);
void schedulerDestroy(void);
void schdExecCallback(SExecResult* pResult, void* param, int32_t code);
#ifdef __cplusplus
}
#endif

28
include/libs/stream/tstreamUpdate.h
View File

@ -15,29 +15,29 @@
#ifndef _TSTREAMUPDATE_H_
#define _TSTREAMUPDATE_H_
#include "storageapi.h"
#include "taosdef.h"
#include "tarray.h"
#include "tcommon.h"
#include "tmsg.h"
#include "storageapi.h"
#ifdef __cplusplus
extern "C" {
#endif
SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo *pInfo);
void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo);
void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo);
int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo);
int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo);
void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count);
void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count);
bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo* pInfo);
void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo);
void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo);
int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count);
int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count);
bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
#ifdef __cplusplus
}

3
include/libs/sync/sync.h
View File

@ -70,7 +70,6 @@ typedef int64_t SyncIndex;
typedef int64_t SyncTerm;
typedef struct SSyncNode SSyncNode;
typedef struct SWal SWal;
typedef struct SSyncRaftEntry SSyncRaftEntry;
typedef enum {
@ -238,7 +237,7 @@ typedef struct SSyncInfo {
int32_t batchSize;
SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN];
SWal* pWal;
struct SWal* pWal;
SSyncFSM* pFsm;
SMsgCb* msgcb;
int32_t pingMs;

5
include/libs/transport/thttp.h
View File

@ -28,6 +28,11 @@ typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag;
int32_t taosSendHttpReport(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen,
EHttpCompFlag flag);
int64_t taosInitHttpChan();
int32_t taosSendHttpReportByChan(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen,
EHttpCompFlag flag, int64_t chanId);
void taosDestroyHttpChan(int64_t chanId);
#ifdef __cplusplus
}
#endif

2
include/os/osMemory.h
View File

@ -53,6 +53,8 @@ void taosPrintBackTrace();
void taosMemoryTrim(int32_t size);
void *taosMemoryMallocAlign(uint32_t alignment, int64_t size);
#define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n))
#define taosMemoryFreeClear(ptr) \
do { \
if (ptr) { \

4
include/os/osString.h
View File

@ -55,10 +55,12 @@ typedef enum { M2C = 0, C2M } ConvType;
#define tstrncpy(dst, src, size) \
do { \
strncpy((dst), (src), (size)); \
(void)strncpy((dst), (src), (size)); \
(dst)[(size)-1] = 0; \
} while (0)
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4);
int64_t taosStr2int64(const char *str);

9
include/util/talgo.h
View File

@ -52,7 +52,7 @@ typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void
* @param param
* @param comparFn
*/
void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn);
int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn);
/**
* Non-recursive quick sort.
@ -98,8 +98,8 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size
* @param maxroot: if heap is max root heap
* @return
*/
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char *buf, bool maxroot);
int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char *buf, bool maxroot);
/**
* sort heap to make sure it is a max/min root heap
@ -114,7 +114,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
* @param maxroot: if heap is max root heap
* @return
*/
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, bool maxroot);
int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
bool maxroot);
#ifdef __cplusplus
}

7
include/util/taoserror.h
View File

@ -89,6 +89,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x0022) //
#define TSDB_CODE_RPC_NETWORK_ERROR TAOS_DEF_ERROR_CODE(0, 0x0023)
#define TSDB_CODE_RPC_NETWORK_BUSY TAOS_DEF_ERROR_CODE(0, 0x0024)
#define TSDB_CODE_HTTP_MODULE_QUIT TAOS_DEF_ERROR_CODE(0, 0x0025)
@ -836,6 +837,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_PAR_TBNAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267E)
#define TSDB_CODE_PAR_TAG_NAME_DUPLICATED TAOS_DEF_ERROR_CODE(0, 0x267F)
#define TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC TAOS_DEF_ERROR_CODE(0, 0x2680)
#define TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2681)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner
@ -940,6 +942,11 @@ int32_t taosGetErrSize();
// UTIL
#define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000)
// AUDIT
#define TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON TAOS_DEF_ERROR_CODE(0, 0x6100)
#define TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD TAOS_DEF_ERROR_CODE(0, 0x6101)
#define TSDB_CODE_AUDIT_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0x6102)
#ifdef __cplusplus
}
#endif

4
include/util/tarray.h
View File

@ -205,7 +205,7 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
void taosArrayClearP(SArray* pArray, void (*fp)(void*));
void* taosArrayDestroy(SArray* pArray);
void taosArrayDestroy(SArray* pArray);
void taosArrayDestroyP(SArray* pArray, FDelete fp);
@ -251,7 +251,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t
* @return
*/
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
int32_t taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver);

22
include/util/tbloomfilter.h
View File

@ -19,6 +19,8 @@
#include "os.h"
#include "tencode.h"
#include "thash.h"
#include "tlog.h"
#include "tutil.h"
#ifdef __cplusplus
extern "C" {
@ -35,19 +37,19 @@ typedef struct SBloomFilter {
uint64_t size;
_hash_fn_t hashFn1;
_hash_fn_t hashFn2;
void *buffer;
void* buffer;
double errorRate;
} SBloomFilter;
SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate);
int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2);
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2);
void tBloomFilterDestroy(SBloomFilter *pBF);
void tBloomFilterDump(const SBloomFilter *pBF);
bool tBloomFilterIsFull(const SBloomFilter *pBF);
int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder);
SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder);
int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF);
int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2);
int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t h1, uint64_t h2);
void tBloomFilterDestroy(SBloomFilter* pBF);
void tBloomFilterDump(const SBloomFilter* pBF);
bool tBloomFilterIsFull(const SBloomFilter* pBF);
int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder);
int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF);
#ifdef __cplusplus
}

4
include/util/tcompare.h
View File

@ -45,7 +45,10 @@ typedef struct SPatternCompareInfo {
TdUcs4 umatchOne; // unicode version matchOne
} SPatternCompareInfo;
int32_t InitRegexCache();
void DestroyRegexCache();
int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo);
int32_t checkRegexPattern(const char *pPattern);
int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo);
@ -83,7 +86,6 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight);
int32_t comparestrRegexMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight);
void DestoryThreadLocalRegComp();
int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight);
int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight);

368
include/util/tencode.h
View File

@ -18,6 +18,7 @@
#include "tcoding.h"
#include "tlist.h"
#include "tutil.h"
#ifdef __cplusplus
extern "C" {
@ -46,38 +47,8 @@ typedef struct {
SDecoderNode* dStack;
} SDecoder;
#define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL)
#define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0]
#define tRPut16(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[0];
#define tRPut32(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[0];
#define tRPut64(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[7]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[6]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[5]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[4]; \
((uint8_t*)(PDEST))[4] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[5] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[6] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[7] = ((uint8_t*)(PSRC))[0];
#define tRGet16 tRPut16
#define tRGet32 tRPut32
#define tRGet64 tRPut64
#define TD_CODER_POS(CODER) ((CODER)->pos)
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos)
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos)
#define tEncodeSize(E, S, SIZE, RET) \
do { \
@ -100,6 +71,7 @@ void tEncoderInit(SEncoder* pCoder, uint8_t* data, uint32_t size);
void tEncoderClear(SEncoder* pCoder);
int32_t tStartEncode(SEncoder* pCoder);
void tEndEncode(SEncoder* pCoder);
static int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size);
static int32_t tEncodeU8(SEncoder* pCoder, uint8_t val);
static int32_t tEncodeI8(SEncoder* pCoder, int8_t val);
static int32_t tEncodeU16(SEncoder* pCoder, uint16_t val);
@ -127,6 +99,7 @@ void tDecoderClear(SDecoder* SDecoder);
int32_t tStartDecode(SDecoder* pCoder);
void tEndDecode(SDecoder* pCoder);
static bool tDecodeIsEnd(SDecoder* pCoder);
static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size);
static int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val);
static int32_t tDecodeI8(SDecoder* pCoder, int8_t* val);
static int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val);
@ -149,97 +122,65 @@ static int32_t tDecodeCStr(SDecoder* pCoder, char** val);
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val);
/* ------------------------ IMPL ------------------------ */
#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) return -1; \
tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL)); \
} \
TD_CODER_MOVE_POS(CODER, sizeof(VAL)); \
return 0;
#define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \
while ((VAL) >= ENCODE_LIMIT) { \
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \
} \
\
(VAL) >>= 7; \
TD_CODER_MOVE_POS(CODER, 1); \
} \
\
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TD_CODER_CURRENT(CODER)[0] = (uint8_t)(VAL); \
} \
TD_CODER_MOVE_POS(CODER, 1); \
return 0;
#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS) \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) return -1; \
tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL)); \
TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \
return 0;
#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \
int32_t i = 0; \
*(PVAL) = 0; \
for (;;) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TYPE tval = TD_CODER_CURRENT(CODER)[0]; \
if (tval < ENCODE_LIMIT) { \
*(PVAL) |= (tval << (7 * i)); \
TD_CODER_MOVE_POS(pCoder, 1); \
break; \
} else { \
*(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \
i++; \
TD_CODER_MOVE_POS(pCoder, 1); \
} \
} \
\
return 0;
// 8
static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) {
static FORCE_INLINE int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size) {
if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1;
tPut(uint8_t, TD_CODER_CURRENT(pCoder), val);
if (pCoder->pos + size > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
memcpy(pCoder->data + pCoder->pos, val, size);
}
TD_CODER_MOVE_POS(pCoder, sizeof(val));
pCoder->pos += size;
return 0;
}
static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) {
if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1;
tPut(int8_t, TD_CODER_CURRENT(pCoder), val);
}
TD_CODER_MOVE_POS(pCoder, sizeof(val));
return 0;
static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); }
static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); }
static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) {
while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
// 16
static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) { TD_ENCODE_MACRO(pCoder, val, uint16_t, 16); }
static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) { TD_ENCODE_MACRO(pCoder, val, int16_t, 16); }
// 32
static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) { TD_ENCODE_MACRO(pCoder, val, uint32_t, 32); }
static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) { TD_ENCODE_MACRO(pCoder, val, int32_t, 32); }
// 64
static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) { TD_ENCODE_MACRO(pCoder, val, uint64_t, 64); }
static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) { TD_ENCODE_MACRO(pCoder, val, int64_t, 64); }
// 16v
static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) {
return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val));
}
// 32v
static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) {
while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) {
return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val));
}
// 64v
static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) {
while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) {
return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val));
}
@ -265,14 +206,16 @@ static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) {
}
static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) {
if (tEncodeU32v(pCoder, len) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeU32v(pCoder, len));
if (len) {
if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) return -1;
memcpy(TD_CODER_CURRENT(pCoder), val, len);
if (pCoder->pos + len > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
memcpy(pCoder->data + pCoder->pos, val, len);
}
TD_CODER_MOVE_POS(pCoder, len);
pCoder->pos += len;
}
return 0;
}
@ -286,70 +229,137 @@ static FORCE_INLINE int32_t tEncodeCStr(SEncoder* pCoder, const char* val) {
}
/* ------------------------ FOR DECODER ------------------------ */
// 8
static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1;
tGet(uint8_t, TD_CODER_CURRENT(pCoder), *val);
TD_CODER_MOVE_POS(pCoder, sizeof(*val));
static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size) {
if (pCoder->pos + size > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} else if (val) {
memcpy(val, pCoder->data + pCoder->pos, size);
}
pCoder->pos += size;
return 0;
}
static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1;
tGet(int8_t, TD_CODER_CURRENT(pCoder), *val);
TD_CODER_MOVE_POS(pCoder, sizeof(*val));
return 0;
static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { return tDecodeFixed(pCoder, val, sizeof(*val)); }
// 16
static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) { TD_DECODE_MACRO(pCoder, val, uint16_t, 16); }
static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) { TD_DECODE_MACRO(pCoder, val, int16_t, 16); }
static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 32
static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) { TD_DECODE_MACRO(pCoder, val, uint32_t, 32); }
static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) { TD_DECODE_MACRO(pCoder, val, int32_t, 32); }
static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 64
static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) { TD_DECODE_MACRO(pCoder, val, uint64_t, 64); }
static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) { TD_DECODE_MACRO(pCoder, val, int64_t, 64); }
static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 16v
static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint16_t);
uint8_t byte;
uint16_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint16_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint16_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
}
static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) {
uint16_t tval;
if (tDecodeU16v(pCoder, &tval) < 0) {
return -1;
TAOS_CHECK_RETURN(tDecodeU16v(pCoder, &tval));
if (val) {
*val = ZIGZAGD(int16_t, tval);
}
if (val) *val = ZIGZAGD(int16_t, tval);
return 0;
}
// 32v
static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint32_t);
uint8_t byte;
uint32_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint32_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint32_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
}
static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) {
uint32_t tval;
if (tDecodeU32v(pCoder, &tval) < 0) {
return -1;
TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &tval));
if (val) {
*val = ZIGZAGD(int32_t, tval);
}
if (val) *val = ZIGZAGD(int32_t, tval);
return 0;
}
// 64v
static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint64_t);
uint8_t byte;
uint64_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint64_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint64_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
}
static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) {
uint64_t tval;
if (tDecodeU64v(pCoder, &tval) < 0) {
return -1;
TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &tval));
if (val) {
*val = ZIGZAGD(int64_t, tval);
}
if (val) *val = ZIGZAGD(int64_t, tval);
return 0;
}
@ -359,11 +369,11 @@ static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) {
float f;
} v;
if (tDecodeU32(pCoder, &(v.ui)) < 0) {
return -1;
}
TAOS_CHECK_RETURN(tDecodeU32(pCoder, &(v.ui)));
*val = v.f;
if (val) {
*val = v.f;
}
return 0;
}
@ -373,30 +383,36 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) {
double d;
} v;
if (tDecodeU64(pCoder, &(v.ui)) < 0) {
return -1;
}
TAOS_CHECK_RETURN(tDecodeU64(pCoder, &(v.ui)));
*val = v.d;
if (val) {
*val = v.d;
}
return 0;
}
static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) {
uint32_t length = 0;
if (tDecodeU32v(pCoder, &length) < 0) return -1;
if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1;
if (val) {
*val = (uint8_t*)TD_CODER_CURRENT(pCoder);
TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length));
if (len) {
*len = length;
}
TD_CODER_MOVE_POS(pCoder, length);
if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
if (val) {
*val = pCoder->data + pCoder->pos;
}
pCoder->pos += length;
return 0;
}
static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) {
if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeBinary(pCoder, (uint8_t**)val, len));
(*len) -= 1;
return 0;
}
@ -409,7 +425,7 @@ static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) {
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
char* pStr;
uint32_t len;
if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len));
memcpy(val, pStr, len + 1);
return 0;
@ -417,16 +433,22 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) {
uint64_t length = 0;
if (tDecodeU64v(pCoder, &length) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &length));
if (length) {
if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1;
*val = taosMemoryMalloc(length);
if (*val == NULL) return -1;
memcpy(*val, TD_CODER_CURRENT(pCoder), length);
if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
TD_CODER_MOVE_POS(pCoder, length);
*val = taosMemoryMalloc(length);
if (*val == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*val, pCoder->data + pCoder->pos, length);
pCoder->pos += length;
} else {
*val = NULL;
}
@ -435,16 +457,20 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin
static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) {
uint32_t length = 0;
if (tDecodeU32v(pCoder, &length) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length));
if (length) {
if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1;
if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
*val = taosMemoryMalloc(length);
if (*val == NULL) return -1;
memcpy(*val, TD_CODER_CURRENT(pCoder), length);
if (*val == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*val, pCoder->data + pCoder->pos, length);
TD_CODER_MOVE_POS(pCoder, length);
pCoder->pos += length;
} else {
*val = NULL;
}
@ -452,7 +478,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u
}
static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) {
if (tDecodeBinaryAlloc(pCoder, (void**)val, len) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeBinaryAlloc(pCoder, (void**)val, len));
(*len) -= 1;
return 0;
}

24
include/util/tscalablebf.h
View File

@ -23,22 +23,22 @@ extern "C" {
#endif
typedef struct SScalableBf {
SArray *bfArray; // array of bloom filters
uint32_t growth;
uint64_t numBits;
uint32_t maxBloomFilters;
int8_t status;
SArray* bfArray; // array of bloom filters
uint32_t growth;
uint64_t numBits;
uint32_t maxBloomFilters;
int8_t status;
_hash_fn_t hashFn1;
_hash_fn_t hashFn2;
} SScalableBf;
SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate);
int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf *pSBf);
int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder);
SScalableBf *tScalableBfDecode(SDecoder *pDecoder);
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf);
int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf* pSBf);
int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder);
int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf);
#ifdef __cplusplus
}

31
include/util/tutil.h
View File

@ -21,6 +21,7 @@
#include "tdef.h"
#include "thash.h"
#include "tmd5.h"
#include "tutil.h"
#ifdef __cplusplus
extern "C" {
@ -56,7 +57,7 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str);
void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen);
int32_t parseCfgReal(const char* str, double* out);
int32_t parseCfgReal(const char *str, double *out);
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
T_MD5_CTX context;
@ -84,9 +85,9 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
static FORCE_INLINE int32_t taosCreateMD5Hash(char *pBuf, int32_t len) {
T_MD5_CTX ctx;
tMD5Init(&ctx);
tMD5Update(&ctx, (uint8_t*)pBuf, len);
tMD5Update(&ctx, (uint8_t *)pBuf, len);
tMD5Final(&ctx);
char* p = pBuf;
char *p = pBuf;
int32_t resLen = 0;
for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) {
resLen += snprintf(p, 3, "%02x", ctx.digest[i]);
@ -147,6 +148,30 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
#define TCONTAINER_OF(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member)))
#define TAOS_RETURN(code) \
do { \
return (terrno = (code)); \
} while (0)
#define TAOS_CHECK_RETURN(CMD) \
do { \
int32_t code = (CMD); \
if (code != TSDB_CODE_SUCCESS) { \
TAOS_RETURN(code); \
} \
} while (0)
#define TAOS_CHECK_GOTO(CMD, LINO, LABEL) \
do { \
code = (CMD); \
if (code != TSDB_CODE_SUCCESS) { \
if (LINO) { \
*((int32_t *)(LINO)) = __LINE__; \
} \
goto LABEL; \
} \
} while (0)
#ifdef __cplusplus
}
#endif

18
source/client/inc/clientSml.h
View File

@ -82,7 +82,7 @@ extern "C" {
#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH))
#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
#define MOVE_FORWARD_ONE(sql, len) ((void)memmove((void *)((sql)-1), (sql), len))
#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \
@ -211,19 +211,19 @@ extern int64_t smlFactorS[];
typedef int32_t (*_equal_fn_sml)(const void *, const void *);
SSmlHandle *smlBuildSmlInfo(TAOS *taos);
int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle);
void smlDestroyInfo(SSmlHandle *info);
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen);
SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat);
int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo);
int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta** sMeta);
int32_t smlSetCTableName(SSmlTableInfo *oneTable);
void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo);
STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen);
int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo);
int32_t smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen, STableMeta **pTableMeta);
int32_t is_same_child_table_telnet(const void *a, const void *b);
int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len);
int32_t smlClearForRerun(SSmlHandle *info);
@ -236,7 +236,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseJSON(SSmlHandle *info, char *payload);
SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement);
int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta);
bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);

2
source/client/src/clientEnv.c
View File

@ -127,7 +127,7 @@ static void generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_
cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code)));
cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType));
cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows));
if(strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){
if(pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){
char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen];
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0';
cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr));

3
source/client/src/clientHb.c
View File

@ -1398,7 +1398,8 @@ void hbMgrCleanUp() {
taosThreadMutexLock(&clientHbMgr.lock);
appHbMgrCleanup();
clientHbMgr.appHbMgrs = taosArrayDestroy(clientHbMgr.appHbMgrs);
taosArrayDestroy(clientHbMgr.appHbMgrs);
clientHbMgr.appHbMgrs = NULL;
taosThreadMutexUnlock(&clientHbMgr.lock);
}

23
source/client/src/clientRawBlockWrite.c
View File

@ -1572,9 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
}
// uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid);
pQuery = smlInitHandle();
if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = smlInitHandle(&pQuery);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
@ -1656,9 +1655,9 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
pQuery = smlInitHandle();
if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = smlInitHandle(&pQuery);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
@ -1755,12 +1754,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle();
if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = smlInitHandle(&pQuery);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("init sml handle failed");
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter);
@ -1903,12 +1902,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle();
if (pQuery == NULL) {
code = smlInitHandle(&pQuery);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("init sml handle failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum);

637
source/client/src/clientSml.c
View File

File diff suppressed because it is too large Load Diff

118
source/client/src/clientSmlJson.c
View File

@ -29,29 +29,29 @@
(start)++; \
}
static char *smlJsonGetObj(char *payload) {
static int32_t smlJsonGetObj(char **payload) {
int leftBracketCnt = 0;
bool isInQuote = false;
while (*payload) {
if (*payload == '"' && *(payload - 1) != '\\') {
while (**payload) {
if (**payload == '"' && *((*payload) - 1) != '\\') {
isInQuote = !isInQuote;
} else if (!isInQuote && unlikely(*payload == '{')) {
} else if (!isInQuote && unlikely(**payload == '{')) {
leftBracketCnt++;
payload++;
(*payload)++;
continue;
} else if (!isInQuote && unlikely(*payload == '}')) {
} else if (!isInQuote && unlikely(**payload == '}')) {
leftBracketCnt--;
payload++;
(*payload)++;
if (leftBracketCnt == 0) {
return payload;
return 0;
} else if (leftBracketCnt < 0) {
return NULL;
return -1;
}
continue;
}
payload++;
(*payload)++;
}
return NULL;
return -1;
}
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
@ -99,8 +99,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
offset[index++] = *start - sTmp;
element->timestamp = (*start);
if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start);
*start = tmp;
}
@ -127,8 +128,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
offset[index++] = *start - sTmp;
element->cols = (*start);
if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start);
*start = tmp;
}
@ -153,8 +155,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
JUMP_JSON_SPACE((*start))
offset[index++] = *start - sTmp;
element->tags = (*start);
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start);
*start = tmp;
}
@ -209,8 +212,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
(*start) += offset[index++];
element->timestamp = *start;
if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start);
*start = tmp;
}
@ -227,8 +231,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
(*start) += offset[index++];
element->cols = *start;
if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start);
*start = tmp;
}
@ -244,8 +249,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
} else if ((*start)[1] == 't' && (*start)[2] == 'a') {
(*start) += offset[index++];
element->tags = (*start);
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start);
*start = tmp;
}
@ -261,7 +267,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
uError("elements != %d", OTD_JSON_FIELDS_NUM);
return TSDB_CODE_TSC_INVALID_JSON;
}
return 0;
return TSDB_CODE_SUCCESS;
}
static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) {
@ -466,7 +472,11 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
break;
}
case cJSON_String: {
smlConvertJSONString(kv, "binary", root);
int32_t ret = smlConvertJSONString(kv, "binary", root);
if (ret != TSDB_CODE_SUCCESS) {
uError("OTD:Failed to parse binary value from JSON Obj");
return ret;
}
break;
}
case cJSON_Object: {
@ -492,20 +502,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
int32_t tagNum = cJSON_GetArraySize(tags);
if (unlikely(tagNum == 0)) {
uError("SML:Tag should not be empty");
terrno = TSDB_CODE_TSC_INVALID_JSON;
return -1;
return TSDB_CODE_TSC_INVALID_JSON;
}
for (int32_t i = 0; i < tagNum; ++i) {
cJSON *tag = cJSON_GetArrayItem(tags, i);
if (unlikely(tag == NULL)) {
terrno = TSDB_CODE_TSC_INVALID_JSON;
return -1;
return TSDB_CODE_TSC_INVALID_JSON;
}
size_t keyLen = strlen(tag->string);
if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
uError("OTD:Tag key length is 0 or too large than 64");
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
// add kv to SSmlKv
@ -516,19 +523,19 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
// value
int32_t ret = smlParseValueFromJSON(tag, &kv);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
terrno = ret;
return -1;
return ret;
}
if (taosArrayPush(preLineKV, &kv) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(preLineKV, &kv);
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS;
return -1;
return TSDB_CODE_TSC_INVALID_JSON;
}
cnt++;
}
return 0;
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
@ -536,12 +543,18 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
if(info->dataFormat){
ret = smlProcessSuperTable(info, elements);
if(ret != 0){
return terrno;
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
}
ret = smlProcessTagJson(info, tags);
if(ret != 0){
return terrno;
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
ret = smlJoinMeasureTag(elements);
if(ret != 0){
@ -703,6 +716,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
// Parse tags
bool needFree = info->dataFormat;
elements->tags = cJSON_PrintUnformatted(tagsJson);
if (elements->tags == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
elements->tagsLen = strlen(elements->tags);
if (is_same_child_table_telnet(elements, &info->preLine) != 0) {
ret = smlParseTagsFromJSON(info, tagsJson, elements);
@ -729,7 +745,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
// notice!!! put ts back to tag to ensure get meta->precision
int64_t ts = smlParseTSFromJSON(info, tsJson);
if (unlikely(ts < 0)) {
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id);
char* tmp = cJSON_PrintUnformatted(tsJson);
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts);
return TSDB_CODE_INVALID_TIMESTAMP;
}
SSmlKv kvTs = {0};
@ -823,7 +840,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
}
if (ret != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_VALUE;
return ret;
}
if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS;
@ -844,18 +861,23 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
cJSON *valueJson = cJSON_Parse(elements->cols);
if (unlikely(valueJson == NULL)) {
uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_JSON;
}
taosArrayPush(info->tagJsonArray, &valueJson);
if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){
cJSON_Delete(valueJson);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_OUT_OF_MEMORY;
}
ret = smlParseValueFromJSONObj(valueJson, &kv);
if (ret != TSDB_CODE_SUCCESS) {
uError("SML:Failed to parse value from JSON Obj:%s", elements->cols);
uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_VALUE;
}
elements->cols[elements->colsLen] = tmp;
} else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) {
uError("SML:cols invalidate:%s", elements->cols);
uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols);
return TSDB_CODE_TSC_INVALID_VALUE;
}
@ -870,7 +892,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
return TSDB_CODE_TSC_INVALID_JSON;
}
taosArrayPush(info->tagJsonArray, &tagsJson);
if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){
cJSON_Delete(tagsJson);
uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id);
return TSDB_CODE_OUT_OF_MEMORY;
}
ret = smlParseTagsFromJSON(info, tagsJson, elements);
if (unlikely(ret)) {
uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id);
@ -937,7 +963,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) {
return ret;
}
info->lines = (SSmlLineInfo *)tmp;
memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo));
(void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo));
}
ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt);
if ((info->lines + cnt)->measure == NULL) break;

68
source/client/src/clientSmlLine.c
View File

@ -103,7 +103,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
return code;
}
char* tmp = taosMemoryCalloc(pVal->length, 1);
memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN);
code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length);
taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) {
@ -149,7 +152,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
if(data == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length);
(void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length);
pVal->value = data;
}
@ -212,8 +215,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
while (*sql < sqlEnd) {
if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(IS_EQUAL(*sql,escapeChar))) {
keyLen = *sql - key;
@ -230,8 +232,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
// parse value
@ -245,8 +246,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
break;
} else if (unlikely(IS_EQUAL(*sql,escapeChar))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
@ -261,24 +261,28 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
if (unlikely(valueLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
return -1;
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
if (keyEscaped) {
char *tmp = (char *)taosMemoryMalloc(keyLen);
memcpy(tmp, key, keyLen);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, key, keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen);
key = tmp;
}
if (valueEscaped) {
char *tmp = (char *)taosMemoryMalloc(valueLen);
memcpy(tmp, value, valueLen);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, value, valueLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen);
value = tmp;
}
@ -289,11 +293,12 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
.length = valueLen,
.keyEscaped = keyEscaped,
.valueEscaped = valueEscaped};
taosArrayPush(preLineKV, &kv);
if(taosArrayPush(preLineKV, &kv) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS;
return -1;
return TSDB_CODE_TSC_INVALID_JSON;
}
cnt++;
@ -302,7 +307,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
}
(*sql)++;
}
return 0;
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
@ -315,13 +320,19 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if(info->dataFormat){
ret = smlProcessSuperTable(info, elements);
if(ret != 0){
return terrno;
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
}
ret = smlProcessTagLine(info, sql, sqlEnd);
if(ret != 0){
return terrno;
if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
return smlProcessChildTable(info, elements);
@ -410,7 +421,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if (keyEscaped) {
char *tmp = (char *)taosMemoryMalloc(kv.keyLen);
memcpy(tmp, key, kv.keyLen);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, key, kv.keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen);
kv.key = tmp;
kv.keyEscaped = keyEscaped;
@ -418,7 +432,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if (valueEscaped) {
char *tmp = (char *)taosMemoryMalloc(kv.length);
memcpy(tmp, kv.value, kv.length);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, kv.value, kv.length);
PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY);
if(kv.type == TSDB_DATA_TYPE_VARBINARY){
@ -437,8 +454,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
} else {
if (currElement->colArray == NULL) {
currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
if (currElement->colArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(currElement->colArray, &kv); // reserve for timestamp
}
cnt++;

34
source/client/src/clientSmlTelnet.c
View File

@ -84,8 +84,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
while (sql < sqlEnd) {
if (unlikely(*sql == SPACE)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(*sql == EQUAL)) {
keyLen = sql - key;
@ -97,8 +96,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
// parse value
@ -111,8 +109,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
}
if (unlikely(*sql == EQUAL)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
sql++;
}
@ -120,13 +117,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
if (unlikely(valueLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
terrno = TSDB_CODE_TSC_INVALID_VALUE;
return -1;
return TSDB_CODE_TSC_INVALID_VALUE;
}
if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
return -1;
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
SSmlKv kv = {.key = key,
@ -136,14 +131,15 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
.length = valueLen,
.keyEscaped = false,
.valueEscaped = false};
taosArrayPush(preLineKV, &kv);
if (taosArrayPush(preLineKV, &kv) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
cnt++;
}
return 0;
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
@ -156,13 +152,19 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
if(info->dataFormat){
ret = smlProcessSuperTable(info, elements);
if(ret != 0){
return terrno;
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
}
ret = smlProcessTagTelnet(info, data, sqlEnd);
if(ret != 0){
return terrno;
if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
ret = smlJoinMeasureTag(elements);

114
source/client/test/smlTest.cpp
View File

@ -40,13 +40,16 @@ TEST(testCase, smlParseInfluxString_Test) {
msgBuf.len = 256;
SSmlLineInfo elements = {0};
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, 0);
info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false;
// case 1
char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3";
char *sql = (char *)taosMemoryCalloc(256, 1);
memcpy(sql, tmp, strlen(tmp) + 1);
ASSERT_NE(sql, nullptr);
(void)memcpy(sql, tmp, strlen(tmp) + 1);
int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql);
@ -62,22 +65,22 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 2 false
tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 4 tag is null
tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql);
@ -92,13 +95,13 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 5 tag is null
tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql + 1);
@ -112,36 +115,36 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 6
tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0);
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
smlClearForRerun(info);
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
ASSERT_EQ(smlClearForRerun(info), 0);
// case 7
tmp = " st , ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 8 false
tmp = ", st , ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
taosMemoryFree(sql);
smlDestroyInfo(info);
@ -228,7 +231,9 @@ TEST(testCase, smlParseCols_Error_Test) {
"escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000"
};
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false;
for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) {
@ -238,13 +243,14 @@ TEST(testCase, smlParseCols_Error_Test) {
msgBuf.len = 256;
int32_t len = strlen(data[i]);
char *sql = (char *)taosMemoryCalloc(256, 1);
memcpy(sql, data[i], len + 1);
ASSERT_NE(sql, nullptr);
(void)memcpy(sql, data[i], len + 1);
SSmlLineInfo elements = {0};
int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements);
// printf("i:%d\n", i);
ASSERT_NE(ret, TSDB_CODE_SUCCESS);
taosMemoryFree(sql);
taosArrayDestroy(elements.colArray);
(void)taosArrayDestroy(elements.colArray);
}
smlDestroyInfo(info);
}
@ -254,7 +260,9 @@ TEST(testCase, smlParseCols_Test) {
SSmlMsgBuf msgBuf;
msgBuf.buf = msg;
msgBuf.len = 256;
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false;
SSmlLineInfo elements = {0};
@ -267,7 +275,8 @@ TEST(testCase, smlParseCols_Test) {
"cboolf=f,cnch_=l\"iuwq\" 1626006833639000000";
int32_t len = strlen(data);
char *sql = (char *)taosMemoryCalloc(1024, 1);
memcpy(sql, data, len + 1);
ASSERT_NE(sql, nullptr);
(void)memcpy(sql, data, len + 1);
int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements);
ASSERT_EQ(ret, TSDB_CODE_SUCCESS);
int32_t size = taosArrayGetSize(elements.colArray);
@ -275,6 +284,7 @@ TEST(testCase, smlParseCols_Test) {
// binary
SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0);
ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY);
@ -285,6 +295,7 @@ TEST(testCase, smlParseCols_Test) {
// nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 2);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
@ -293,6 +304,7 @@ TEST(testCase, smlParseCols_Test) {
// bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 3);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0);
ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -301,6 +313,7 @@ TEST(testCase, smlParseCols_Test) {
// double
kv = (SSmlKv *)taosArrayGet(elements.colArray, 4);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
@ -310,6 +323,7 @@ TEST(testCase, smlParseCols_Test) {
// float
kv = (SSmlKv *)taosArrayGet(elements.colArray, 5);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0);
ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
@ -319,6 +333,7 @@ TEST(testCase, smlParseCols_Test) {
// float
kv = (SSmlKv *)taosArrayGet(elements.colArray, 6);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT);
@ -328,6 +343,7 @@ TEST(testCase, smlParseCols_Test) {
// tiny int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 7);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0);
ASSERT_EQ(kv->keyLen, 3);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT);
@ -336,6 +352,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned tiny int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 8);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0);
ASSERT_EQ(kv->keyLen, 3);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT);
@ -344,6 +361,7 @@ TEST(testCase, smlParseCols_Test) {
// small int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 9);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT);
@ -352,6 +370,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned smallint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 10);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT);
@ -360,6 +379,7 @@ TEST(testCase, smlParseCols_Test) {
// int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 11);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT);
@ -368,6 +388,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 12);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT);
@ -376,6 +397,7 @@ TEST(testCase, smlParseCols_Test) {
// bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 13);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT);
@ -384,6 +406,7 @@ TEST(testCase, smlParseCols_Test) {
// bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 14);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0);
ASSERT_EQ(kv->keyLen, 2);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT);
@ -392,6 +415,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 15);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT);
@ -400,6 +424,7 @@ TEST(testCase, smlParseCols_Test) {
// bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 16);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0);
ASSERT_EQ(kv->keyLen, 9);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -408,6 +433,7 @@ TEST(testCase, smlParseCols_Test) {
// bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 17);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0);
ASSERT_EQ(kv->keyLen, 6);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -416,6 +442,7 @@ TEST(testCase, smlParseCols_Test) {
// bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 18);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0);
ASSERT_EQ(kv->keyLen, 6);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -424,13 +451,14 @@ TEST(testCase, smlParseCols_Test) {
// nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 19);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0);
ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->length, 4);
ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0);
taosArrayDestroy(elements.colArray);
(void)taosArrayDestroy(elements.colArray);
taosMemoryFree(sql);
smlDestroyInfo(info);
}
@ -471,7 +499,9 @@ TEST(testCase, smlParseNumber_Test) {
}
TEST(testCase, smlParseTelnetLine_error_Test) {
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->dataFormat = false;
info->protocol = TSDB_SML_TELNET_PROTOCOL;
ASSERT_NE(info, nullptr);
@ -509,7 +539,9 @@ TEST(testCase, smlParseTelnetLine_error_Test) {
}
TEST(testCase, smlParseTelnetLine_Test) {
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->dataFormat = false;
info->protocol = TSDB_SML_TELNET_PROTOCOL;
ASSERT_NE(info, nullptr);
@ -528,7 +560,9 @@ TEST(testCase, smlParseTelnetLine_Test) {
}
//TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) {
// SSmlHandle *info = smlBuildSmlInfo(NULL);
// SSmlHandle *info = nullptr;
// int32_t code = smlBuildSmlInfo(nullptr, &info);
// ASSERT_EQ(code, TSDB_CODE_SUCCESS);
// info->protocol = TSDB_SML_JSON_PROTOCOL;
// ASSERT_NE(info, nullptr);
//
@ -568,7 +602,7 @@ TEST(testCase, smlParseNumber_performance_Test) {
for (int j = 0; j < 10000000; ++j) {
kv.value = str[i];
kv.length = strlen(str[i]);
smlParseNumber(&kv, &msgBuf);
(void)smlParseNumber(&kv, &msgBuf); //ignore result
}
printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1);
printf("\n");
@ -576,7 +610,7 @@ TEST(testCase, smlParseNumber_performance_Test) {
for (int j = 0; j < 10000000; ++j) {
kv.value = str[i];
kv.length = strlen(str[i]);
smlParseNumberOld(&kv, &msgBuf);
(void)smlParseNumberOld(&kv, &msgBuf); //ignore result
}
printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2);
printf("\n\n");

183
source/common/src/cos.c
View File

@ -3,6 +3,7 @@
#include "cos.h"
#include "cos_cp.h"
#include "tdef.h"
#include "tutil.h"
extern char tsS3Endpoint[];
extern char tsS3AccessKeyId[];
@ -40,7 +41,7 @@ int32_t s3Begin() {
if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) {
uError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
protocolG = !tsS3Https;
@ -48,12 +49,12 @@ int32_t s3Begin() {
uriStyleG = S3UriStyleVirtualHost;
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
void s3End() { S3_deinitialize(); }
int32_t s3Init() { return 0; /*s3Begin();*/ }
int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); /*s3Begin();*/ }
void s3CleanUp() { /*s3End();*/
}
@ -61,7 +62,7 @@ void s3CleanUp() { /*s3End();*/
static int32_t s3ListBucket(char const *bucketname);
int32_t s3CheckCfg() {
int32_t code = 0;
int32_t code = 0, lino = 0;
if (!tsS3Enabled) {
fprintf(stderr, "s3 not configured.\n");
@ -71,7 +72,7 @@ int32_t s3CheckCfg() {
code = s3Begin();
if (code != 0) {
fprintf(stderr, "failed to initialize s3.\n");
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
// test put
@ -91,20 +92,17 @@ int32_t s3CheckCfg() {
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (!fp) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to open test file: %s.\n", path);
// uError("ERROR: %s Failed to open %s", __func__, path);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to write test file: %s.\n", path);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to fsync test file: %s.\n", path);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
taosCloseFile(&fp);
@ -112,7 +110,7 @@ int32_t s3CheckCfg() {
code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16);
if (code != 0) {
fprintf(stderr, "put object %s : failed.\n", objectname[0]);
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
fprintf(stderr, "put object %s: success.\n\n", objectname[0]);
@ -121,7 +119,7 @@ int32_t s3CheckCfg() {
code = s3ListBucket(tsS3BucketName);
if (code != 0) {
fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName);
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName);
@ -134,7 +132,7 @@ int32_t s3CheckCfg() {
code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
if (code != 0) {
fprintf(stderr, "get object %s : failed.\n", objectname[0]);
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
char buf[7] = {0};
memcpy(buf, pBlock, c_len);
@ -147,14 +145,14 @@ int32_t s3CheckCfg() {
code = s3DeleteObjects(objectname, 1);
if (code != 0) {
fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
fprintf(stderr, "delete object %s: success.\n\n", objectname[0]);
s3End();
_exit:
return code;
TAOS_RETURN(code);
}
static int should_retry() {
@ -169,7 +167,7 @@ static int should_retry() {
}
*/
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status,
@ -202,7 +200,6 @@ static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *prope
}
static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) {
//(void)callbackData;
TS3SizeCBD *cbd = callbackData;
if (properties->contentLength > 0) {
cbd->content_length = properties->contentLength;
@ -249,7 +246,7 @@ static int32_t s3ListBucket(char const *bucketname) {
SArray *objectArray = getListByPrefix("s3");
if (objectArray == NULL) {
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
const char **object_name = TARRAY_DATA(objectArray);
@ -262,7 +259,7 @@ static int32_t s3ListBucket(char const *bucketname) {
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return code;
TAOS_RETURN(code);
}
typedef struct growbuffer {
@ -607,13 +604,13 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c
code = TAOS_SYSTEM_ERROR(EIO);
}
return code;
TAOS_RETURN(code);
}
static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name,
int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) {
int32_t code = 0;
int32_t code = 0, lino = 0;
uint64_t totalContentLength = contentLength;
uint64_t todoContentLength = contentLength;
UploadManager manager = {0};
@ -640,6 +637,9 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
if (!manager.etags) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
manager.next_etags_pos = 0;
do {
S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
@ -647,8 +647,7 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
upload:
@ -672,8 +671,7 @@ upload:
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
contentLength -= chunk_size;
todoContentLength -= chunk_size;
@ -686,8 +684,7 @@ upload:
int n;
for (i = 0; i < totalSeq; i++) {
if (!manager.etags[i]) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
n = snprintf(buf, sizeof(buf),
"<Part><PartNumber>%d</PartNumber>"
@ -704,11 +701,13 @@ upload:
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
clean:
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (manager.upload_id) {
taosMemoryFree(manager.upload_id);
}
@ -718,13 +717,13 @@ clean:
growbuffer_destroy(manager.gb);
taosMemoryFree(manager.etags);
return code;
TAOS_RETURN(code);
}
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) {
int32_t code = 0;
int32_t code = 0, lino = 0;
uint64_t totalContentLength = contentLength;
// uint64_t todoContentLength = contentLength;
@ -744,13 +743,16 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
SCheckpoint cp = {0};
cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
if (!cp.parts) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (taosCheckExistFile(file_cp_path)) {
if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) {
manager.upload_id = strdup(cp.upload_id);
need_init_upload = false;
} else {
cos_cp_remove(file_cp_path);
TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit);
}
}
@ -763,16 +765,14 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size);
}
if (cos_cp_open(file_cp_path, &cp)) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
int part_num = 0;
@ -791,6 +791,10 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
if (!manager.etags) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
manager.next_etags_pos = 0;
upload:
@ -802,8 +806,7 @@ upload:
if (i > 0 && cp.parts[i - 1].completed) {
if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
}
@ -827,25 +830,23 @@ upload:
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
//(void)cos_cp_dump(&cp);
goto clean;
}
if (!manager.etags[seq - 1]) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0);
(void)cos_cp_dump(&cp);
TAOS_CHECK_GOTO(cos_cp_dump(&cp), &lino, _exit);
contentLength -= chunk_size;
// todoContentLength -= chunk_size;
}
cos_cp_close(cp.thefile);
TAOS_CHECK_GOTO(cos_cp_close(cp.thefile), &lino, _exit);
cp.thefile = 0;
int size = 0;
@ -869,20 +870,23 @@ upload:
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
cos_cp_remove(file_cp_path);
TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit);
clean:
_exit:
/*
if (parts) {
taosMemoryFree(parts);
}
*/
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (cp.thefile) {
cos_cp_close(cp.thefile);
(void)cos_cp_close(cp.thefile);
}
if (cp.parts) {
taosMemoryFree(cp.parts);
@ -899,7 +903,7 @@ clean:
taosMemoryFree(manager.etags);
growbuffer_destroy(manager.gb);
return code;
TAOS_RETURN(code);
}
int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) {
@ -917,15 +921,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
@ -956,7 +958,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
growbuffer_destroy(data.gb);
}
return code;
TAOS_RETURN(code);
}
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
@ -974,22 +976,19 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int
put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
contentLength = size;
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) {
taosCloseFile(&data.infileFD);
code = TAOS_SYSTEM_ERROR(errno);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
@ -1016,7 +1015,7 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int
growbuffer_destroy(data.gb);
}
return code;
TAOS_RETURN(code);
}
typedef struct list_bucket_callback_data {
@ -1130,11 +1129,11 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
code = -1;
code = TSDB_CODE_FAILED;
}
}
return code;
TAOS_RETURN(code);
}
void s3DeleteObjectsByPrefix(const char *prefix) {
@ -1187,17 +1186,19 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
if (cbd.status != S3StatusOK) {
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
}
if (check && cbd.buf_pos != size) {
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
}
*ppBlock = cbd.buf;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) {
@ -1218,9 +1219,8 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) {
TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("[s3] open file error, errno:%d, fileName:%s", terrno, fileName);
return -1;
uError("[s3] open file error, errno:%d, fileName:%s", TAOS_SYSTEM_ERROR(errno), fileName);
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
TS3GetData cbd = {0};
@ -1232,16 +1232,17 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) {
if (cbd.status != S3StatusOK) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
taosCloseFile(&pFile);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
}
taosCloseFile(&pFile);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
SArray *objectArray = getListByPrefix(prefix);
if (objectArray == NULL) return -1;
if (objectArray == NULL) TAOS_RETURN(TSDB_CODE_FAILED);
for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) {
char *object = taosArrayGetP(objectArray, i);
@ -1255,7 +1256,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
}
if (s3GetObjectToFile(object, fileName) != 0) {
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
}
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
@ -1279,7 +1280,7 @@ long s3Size(const char *object_name) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
size = cbd.content_length;
@ -1306,7 +1307,7 @@ int32_t s3Init() {
// set log output, default stderr
cos_log_set_output(NULL);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
void s3CleanUp() { cos_http_io_deinitialize(); }
@ -1363,10 +1364,10 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
cos_pool_destroy(p);
if (s->code != 200) {
return code = s->code;
TAOS_RETURN(s->code);
}
return code;
TAOS_RETURN(code);
}
int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) {
@ -1405,10 +1406,10 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_
cos_pool_destroy(p);
if (s->code != 200) {
return code = s->code;
TAOS_RETURN(s->code);
}
return code;
TAOS_RETURN(code);
}
void s3DeleteObjectsByPrefix(const char *prefix_str) {
@ -1465,7 +1466,7 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) {
cos_warn_log("delete objects failed\n");
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
bool s3Exists(const char *object_name) {
@ -1547,7 +1548,7 @@ bool s3Get(const char *object_name, const char *path) {
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) {
(void)check;
int32_t code = 0;
int32_t code = 0, lino = 0;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
@ -1588,10 +1589,9 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
s = cos_get_object_to_buffer(options, &bucket, &object, headers, NULL, &download_buffer, &resp_headers);
log_status(s);
if (!cos_status_is_ok(s)) {
vError("s3: %d(%s)", s->code, s->error_msg);
vError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
code = TAOS_SYSTEM_ERROR(EIO);
return code;
uError("s3: %d(%s)", s->code, s->error_msg);
uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
}
// print_headers(resp_headers);
@ -1601,6 +1601,10 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { len += cos_buf_size(content); }
// char *buf = cos_pcalloc(p, (apr_size_t)(len + 1));
char *buf = taosMemoryCalloc(1, (apr_size_t)(len));
if (!buf) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
// buf[len] = '\0';
cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) {
size = cos_buf_size(content);
@ -1609,12 +1613,13 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
}
// cos_warn_log("Download data=%s", buf);
_exit:
//销毁内存池
cos_pool_destroy(p);
*ppBlock = buf;
return code;
TAOS_RETURN(code);
}
typedef struct {

172
source/common/src/cos_cp.c
View File

@ -9,42 +9,35 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH);
if (!fd) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open %s", __func__, cp_path);
return code;
uError("%s Failed to open %s", __func__, cp_path);
TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno));
}
checkpoint->thefile = fd;
return code;
TAOS_RETURN(code);
}
void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); }
void cos_cp_remove(char const* filepath) { (void)taosRemoveFile(filepath); }
int32_t cos_cp_close(TdFilePtr fd) { return taosCloseFile(&fd); }
int32_t cos_cp_remove(char const* filepath) { return taosRemoveFile(filepath); }
static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
int32_t code = 0;
int32_t code = 0, lino = 0;
cJSON const* item2 = NULL;
cJSON* json = cJSON_Parse(cp_body);
if (NULL == json) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
}
cJSON const* item = cJSON_GetObjectItem(json, "ver");
if (!cJSON_IsNumber(item) || item->valuedouble != 1) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble);
goto _exit;
TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
}
item = cJSON_GetObjectItem(json, "type");
if (!cJSON_IsNumber(item)) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
}
cp->cp_type = item->valuedouble;
@ -155,40 +148,36 @@ static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
}
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (json) cJSON_Delete(json);
if (cp_body) taosMemoryFree(cp_body);
return code;
TAOS_RETURN(code);
}
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
int32_t code = 0;
int32_t code = 0, lino = 0;
char* cp_body = NULL;
TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ);
if (!fd) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open %s", __func__, filepath);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
int64_t size = -1;
code = taosStatFile(filepath, &size, NULL, NULL);
if (code) {
uError("ERROR: %s Failed to stat %s", __func__, filepath);
goto _exit;
}
TAOS_CHECK_GOTO(taosStatFile(filepath, &size, NULL, NULL), &lino, _exit);
char* cp_body = taosMemoryMalloc(size + 1);
cp_body = taosMemoryMalloc(size + 1);
if (!cp_body) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
int64_t n = taosReadFile(fd, cp_body, size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to read %s", __func__, filepath);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
} else if (n != size) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size);
goto _exit;
TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
}
taosCloseFile(&fd);
cp_body[size] = '\0';
@ -196,184 +185,161 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
return cos_cp_parse_body(cp_body, checkpoint);
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (fd) {
taosCloseFile(&fd);
}
if (cp_body) {
taosMemoryFree(cp_body);
}
return code;
TAOS_RETURN(code);
}
static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
int32_t code = 0;
int32_t code = 0, lino = 0;
char* data = cJSON_PrintUnformatted(json);
if (NULL == data) {
return TSDB_CODE_OUT_OF_MEMORY;
if (!data) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
TdFilePtr fp = checkpoint->thefile;
if (taosFtruncateFile(fp, 0) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosLSeekFile(fp, 0, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosWriteFile(fp, data, strlen(data)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosMemoryFree(data);
return code;
TAOS_RETURN(code);
}
int32_t cos_cp_dump(SCheckpoint* cp) {
int32_t code = 0;
int32_t lino = 0;
int32_t code = 0, lino = 0;
cJSON* ojson = NULL;
cJSON* json = cJSON_CreateObject();
if (!json) return TSDB_CODE_OUT_OF_MEMORY;
if (!json) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (COS_CP_TYPE_UPLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "file");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
} else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "object");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
}
ojson = cJSON_AddObjectToObject(json, "cpparts");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts");
if (!ajson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
for (int i = 0; i < cp->part_num; ++i) {
cJSON* item = cJSON_CreateObject();
if (!item) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
cJSON_AddItemToArray(ajson, item);
if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
}
code = cos_cp_save_json(json, cp);
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(code, &lino, _exit);
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
cJSON_Delete(json);
return code;
TAOS_RETURN(code);
}
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {}

13
source/common/src/tdatablock.c
View File

@ -615,6 +615,10 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex)
int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex;
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index);
if (pColInfoData == NULL) {
return 0;
}
if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) {
return 0;
}
@ -1512,7 +1516,9 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
colDataDestroy(pColInfoData);
}
pBlock->pDataBlock = taosArrayDestroy(pBlock->pDataBlock);
taosArrayDestroy(pBlock->pDataBlock);
pBlock->pDataBlock = NULL;
taosMemoryFreeClear(pBlock->pBlockAgg);
memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
}
@ -2486,7 +2492,10 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha
RandTableName rname = {
.tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname};
buildChildTableName(&rname);
int32_t code = buildChildTableName(&rname);
if(code != TSDB_CODE_SUCCESS){
return code;
}
taosArrayDestroy(tags);
if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) {

76
source/common/src/tdataformat.c
View File

@ -305,11 +305,11 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo
*(int32_t *)(fixed + schema->columns[i].offset) = varlen - fixed - sinfo->tupleFixedSize;
varlen += tPutU32v(varlen, colValArray[colValIndex].value.nData);
if (colValArray[colValIndex].value.nData) {
memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
(void)memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
varlen += colValArray[colValIndex].value.nData;
}
} else {
memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val,
(void)memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val,
tDataTypes[schema->columns[i].type].bytes);
}
} else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL
@ -384,12 +384,12 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c
payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData);
if (colValArray[colValIndex].value.nData > 0) {
memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
(void)memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
}
payloadSize += colValArray[colValIndex].value.nData;
} else {
payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
memcpy(payload + payloadSize, &colValArray[colValIndex].value.val,
(void)memcpy(payload + payloadSize, &colValArray[colValIndex].value.val,
tDataTypes[schema->columns[i].type].bytes);
payloadSize += tDataTypes[schema->columns[i].type].bytes;
}
@ -475,7 +475,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted,
value.nData = infos[iInfo].bind->length[iRow];
value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow;
} else {
memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
(void)memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
infos[iInfo].bind->buffer_length);
}
colVal = COL_VAL_VALUE(infos[iInfo].columnId, value);
@ -509,7 +509,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->cid = pTColumn->colId;
pColVal->value.type = pTColumn->type;
pColVal->flag = CV_FLAG_VALUE;
memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY));
(void)memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY));
return 0;
}
@ -573,7 +573,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = NULL;
}
} else {
memcpy(&pColVal->value.val, pData, pTColumn->bytes);
(void)memcpy(&pColVal->value.val, pData, pTColumn->bytes);
}
}
return 0;
@ -624,7 +624,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset);
pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData);
} else {
memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
(void)memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
}
}
@ -861,7 +861,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.cid = pTColumn->colId;
pIter->cv.value.type = pTColumn->type;
pIter->cv.flag = CV_FLAG_VALUE;
memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY));
(void)memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY));
goto _exit;
}
@ -906,7 +906,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL;
}
} else {
memcpy(&pIter->cv.value.val, pData, pTColumn->bytes);
(void)memcpy(&pIter->cv.value.val, pData, pTColumn->bytes);
}
}
@ -965,7 +965,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL;
}
} else {
memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
(void)memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
}
goto _exit;
}
@ -1288,7 +1288,7 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
key->pks[i].pData = tdata;
key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData);
} else {
memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes);
(void)memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes);
}
}
}
@ -1378,7 +1378,7 @@ FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) {
return 0;
}
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
pDst->ts = pSrc->ts;
pDst->numOfPKs = pSrc->numOfPKs;
@ -1392,12 +1392,10 @@ int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
} else {
pVal->nData = pSrc->pks[i].nData;
ASSERT(pSrc->pks[i].pData != NULL);
memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData);
(void)memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData);
}
}
}
return TSDB_CODE_SUCCESS;
}
// STag ========================================
@ -1528,7 +1526,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
} else {
p = p ? p + n : p;
n += tDataTypes[pTagVal->type].bytes;
if (p) memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes);
if (p) (void)memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes);
}
return n;
@ -1550,7 +1548,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
if (IS_VAR_DATA_TYPE(pTagVal->type)) {
n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData);
} else {
memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes);
(void)memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes);
n += tDataTypes[pTagVal->type].bytes;
}
@ -1661,7 +1659,7 @@ char *tTagValToData(const STagVal *value, bool isJson) {
}
varDataLen(data + typeBytes) = value->nData;
memcpy(varDataVal(data + typeBytes), value->pData, value->nData);
(void)memcpy(varDataVal(data + typeBytes), value->pData, value->nData);
} else {
data = ((char *)&(value->i64)) - typeBytes; // json with type
}
@ -1713,7 +1711,7 @@ bool tTagGet(const STag *pTag, STagVal *pTagVal) {
} else if (c > 0) {
lidx = midx + 1;
} else {
memcpy(pTagVal, &tv, sizeof(tv));
(void)memcpy(pTagVal, &tv, sizeof(tv));
return true;
}
}
@ -1892,7 +1890,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
if (nData) {
code = tRealloc(&pColData->pData, pColData->nData + nData);
if (code) goto _exit;
memcpy(pColData->pData + pColData->nData, pData, nData);
(void)memcpy(pColData->pData + pColData->nData, pData, nData);
pColData->nData += nData;
}
} else {
@ -1900,7 +1898,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes);
if (code) goto _exit;
if (pData) {
memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]);
(void)memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]);
} else {
memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]);
}
@ -2594,7 +2592,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo
}
value.pData = pColData->pData + pColData->aOffset[iVal];
} else {
memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes);
(void)memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes);
}
*pColVal = COL_VAL_VALUE(pColData->cid, value);
}
@ -2692,7 +2690,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal));
(void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal));
break;
case (HAS_VALUE | HAS_NULL | HAS_NONE):
pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal));
@ -2700,7 +2698,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal));
(void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal));
break;
default:
pColData->pBitMap = NULL;
@ -2714,7 +2712,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2);
(void)memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2);
} else {
pColData->aOffset = NULL;
}
@ -2727,7 +2725,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
goto _exit;
}
memcpy(pColData->pData, pColDataFrom->pData, pColData->nData);
(void)memcpy(pColData->pData, pColDataFrom->pData, pColData->nData);
} else {
pColData->pData = NULL;
}
@ -3119,10 +3117,10 @@ static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SCo
pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData;
}
memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow],
(void)memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow],
nData);
} else {
memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow],
(void)memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow],
&pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]);
}
return code;
@ -3361,7 +3359,7 @@ static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd /
ASSERT(0);
} else {
if (iv != iStart) {
memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart],
(void)memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart],
&pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]);
}
memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)],
@ -3579,11 +3577,11 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) {
case (HAS_NULL | HAS_NONE):
case (HAS_VALUE | HAS_NONE):
case (HAS_VALUE | HAS_NULL):
if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal));
if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal));
n += BIT1_SIZE(pColData->nVal);
break;
case (HAS_VALUE | HAS_NULL | HAS_NONE):
if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal));
if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal));
n += BIT2_SIZE(pColData->nVal);
break;
default:
@ -3593,14 +3591,14 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) {
// value
if (pColData->flag & HAS_VALUE) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2);
if (pBuf) (void)memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2);
n += (pColData->nVal << 2);
n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData);
if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData);
if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData);
n += pColData->nData;
} else {
if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData);
if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData);
n += pColData->nData;
}
}
@ -4348,7 +4346,7 @@ int32_t tCompressData(void *input, // input
ASSERT(outputSize >= extraSizeNeeded);
if (info->cmprAlg == NO_COMPRESSION) {
memcpy(output, input, info->originalSize);
(void)memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize;
} else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local;
@ -4385,7 +4383,7 @@ int32_t tCompressData(void *input, // input
} else {
DEFINE_VAR(info->cmprAlg)
if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) {
memcpy(output, input, info->originalSize);
(void)memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize;
return 0;
}
@ -4431,7 +4429,7 @@ int32_t tDecompressData(void *input, // input
if (info->cmprAlg == NO_COMPRESSION) {
ASSERT(info->compressedSize == info->originalSize);
memcpy(output, input, info->compressedSize);
(void)memcpy(output, input, info->compressedSize);
} else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local;
@ -4468,7 +4466,7 @@ int32_t tDecompressData(void *input, // input
} else {
DEFINE_VAR(info->cmprAlg);
if (l1 == L1_DISABLED && l2 == L2_DISABLED) {
memcpy(output, input, info->compressedSize);
(void)memcpy(output, input, info->compressedSize);
return 0;
}
SBuffer local;

167
source/common/src/tmisce.c
View File

@ -15,9 +15,9 @@
#define _DEFAULT_SOURCE
#include "tmisce.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tjson.h"
#include "tdatablock.h"
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = 0;
@ -34,18 +34,26 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = tsServerPort;
}
if (pEp->port <= 0) {
return TSDB_CODE_INVALID_PARA;
}
return 0;
}
void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) {
int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) {
if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) {
return;
return TSDB_CODE_INVALID_PARA;
}
int32_t index = pEpSet->numOfEps;
if (index >= sizeof(pEpSet->eps) / sizeof(pEpSet->eps[0])) {
return TSDB_CODE_OUT_OF_RANGE;
}
tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn));
pEpSet->eps[index].port = port;
pEpSet->numOfEps += 1;
return 0;
}
bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) {
@ -131,75 +139,96 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet) {
return ep;
}
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t bufLen) {
int len = snprintf(pBuf, bufLen, "epset:{");
if (len < 0) {
return -1;
}
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) {
int32_t ret = 0;
int32_t nwrite = 0;
for (int _i = 0; (_i < pEpSet->numOfEps) && (bufLen > len); _i++) {
nwrite = snprintf(pBuf + nwrite, cap, "epset:{");
if (nwrite <= 0 || nwrite >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
cap -= nwrite;
for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) {
int32_t ret = 0;
if (_i == pEpSet->numOfEps - 1) {
ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} else {
ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
}
if (ret < 0) {
return -1;
if (ret <= 0 || ret >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
len += ret;
nwrite += ret;
cap -= ret;
}
if (len < bufLen) {
/*len += */snprintf(pBuf + len, bufLen - len, "}, inUse:%d", pEpSet->inUse);
if (cap <= 0) {
return TSDB_CODE_OUT_OF_BUFFER;
}
ret = snprintf(pBuf + nwrite, cap, "}, inUse:%d", pEpSet->inUse);
if (ret <= 0 || ret >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
return TSDB_CODE_SUCCESS;
}
int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) {
SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return -1;
int32_t code = 0;
SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
char tmp[4096] = {0};
tjsonAddDoubleToObject(pJson, "reportVersion", 1);
TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "reportVersion", 1), NULL, _exit);
tjsonAddIntegerToObject(pJson, "clusterId", clusterId);
tjsonAddIntegerToObject(pJson, "startTime", startTime);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "clusterId", clusterId), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "startTime", startTime), NULL, _exit);
// Do NOT invoke the taosGetFqdn here.
// this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked
// environment. The lock operation by taosGetFqdn may cause this program deadlock.
tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn), NULL, _exit);
tjsonAddIntegerToObject(pJson, "pid", taosGetPId());
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "pid", taosGetPId()), NULL, _exit);
taosGetAppName(tmp, NULL);
tjsonAddStringToObject(pJson, "appName", tmp);
code = taosGetAppName(tmp, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "appName", tmp), NULL, _exit);
if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) {
tjsonAddStringToObject(pJson, "os", tmp);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "os", tmp), NULL, _exit);
} else {
// do nothing
}
float numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) {
tjsonAddStringToObject(pJson, "cpuModel", tmp);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "cpuModel", tmp), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores), NULL, _exit);
} else {
tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores);
TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores), NULL, _exit);
}
snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
int32_t nBytes = snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
if (nBytes <= 9 || nBytes >= sizeof(tmp)) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_RANGE, NULL, _exit);
}
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit);
tjsonAddStringToObject(pJson, "version", version);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo);
tjsonAddStringToObject(pJson, "gitInfo", gitinfo);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", version), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", buildinfo), NULL, _exit);
tjsonAddIntegerToObject(pJson, "crashSig", signum);
tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs());
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", gitinfo), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit);
#ifdef _TD_DARWIN_64
taosLogTraceToBuf(tmp, sizeof(tmp), 4);
@ -209,26 +238,47 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t
taosLogTraceToBuf(tmp, sizeof(tmp), 8);
#endif
tjsonAddStringToObject(pJson, "stackInfo", tmp);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "stackInfo", tmp), NULL, _exit);
char* pCont = tjsonToString(pJson);
if (pCont == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_GOTO(code, NULL, _exit);
goto _exit;
}
tjsonDelete(pJson);
*pMsg = pCont;
return TSDB_CODE_SUCCESS;
pJson = NULL;
_exit:
tjsonDelete(pJson);
TAOS_RETURN(code);
}
int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
SConfig* pConf = taosGetCfg();
int32_t code = 0;
SConfig* pConf = taosGetCfg();
if (pConf == NULL) {
return TSDB_CODE_INVALID_CFG;
}
int32_t numOfRows = 0;
int32_t col = startCol;
SConfigItem* pItem = NULL;
SConfigIter* pIter = NULL;
blockDataEnsureCapacity(pBlock, cfgGetSize(pConf));
SConfigIter* pIter = cfgCreateIter(pConf);
int8_t locked = 0;
TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit);
pIter = cfgCreateIter(pConf);
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
cfgLock(pConf);
locked = 1;
while ((pItem = cfgNextIter(pIter)) != NULL) {
col = startCol;
@ -236,29 +286,44 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
// GRANT_CFG_SKIP;
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, name, false);
if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit);
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
int32_t valueLen = 0;
cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
varDataSetLen(value, valueLen);
pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, value, false);
if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, value, false), NULL, _exit);
char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0};
cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen);
varDataSetLen(scope, valueLen);
pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, scope, false);
if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit);
numOfRows++;
}
cfgUnLock(pConf);
pBlock->info.rows = numOfRows;
_exit:
if (locked) cfgUnLock(pConf);
cfgDestroyIter(pIter);
return TSDB_CODE_SUCCESS;
TAOS_RETURN(code);
}

6
source/common/src/tmsg.c
View File

@ -9505,7 +9505,8 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, void *pRsp) {
static void tDeleteMqDataRspCommon(void *rsp) {
SMqDataRspCommon *pRsp = rsp;
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);
taosArrayDestroy(pRsp->blockDataLen);
pRsp->blockDataLen = NULL;
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
pRsp->blockData = NULL;
taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper);
@ -9558,7 +9559,8 @@ void tDeleteSTaosxRsp(void *rsp) {
tDeleteMqDataRspCommon(rsp);
STaosxRsp *pRsp = (STaosxRsp *)rsp;
pRsp->createTableLen = taosArrayDestroy(pRsp->createTableLen);
taosArrayDestroy(pRsp->createTableLen);
pRsp->createTableLen = NULL;
taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree);
pRsp->createTableReq = NULL;
}

13
source/common/src/tname.c
View File

@ -293,17 +293,19 @@ static int compareKv(const void* p1, const void* p2) {
/*
* use stable name and tags to grearate child table name
*/
void buildChildTableName(RandTableName* rName) {
int32_t buildChildTableName(RandTableName* rName) {
SStringBuilder sb = {0};
taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen);
if (sb.buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return;
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArraySort(rName->tags, compareKv);
for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) {
taosStringBuilderAppendChar(&sb, ',');
SSmlKv* tagKv = taosArrayGet(rName->tags, j);
if(tagKv == NULL) {
return TSDB_CODE_SML_INVALID_DATA;
}
taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
taosStringBuilderAppendChar(&sb, '=');
if (IS_VAR_DATA_TYPE(tagKv->type)) {
@ -323,8 +325,9 @@ void buildChildTableName(RandTableName* rName) {
rName->ctbShortName[0] = 't';
rName->ctbShortName[1] = '_';
for (int i = 0; i < 16; i++) {
sprintf(temp, "%02x", context.digest[i]);
strcat(rName->ctbShortName, temp);
(void)sprintf(temp, "%02x", context.digest[i]);
(void)strcat(rName->ctbShortName, temp);
}
taosStringBuilderDestroy(&sb);
return TSDB_CODE_SUCCESS;
}

4
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -34,12 +34,14 @@ typedef struct SVnodeMgmt {
SAutoQWorkerPool streamPool;
SWWorkerPool fetchPool;
SSingleWorker mgmtWorker;
SSingleWorker mgmtMultiWorker;
SHashObj *hash;
TdThreadRwlock lock;
SVnodesStat state;
STfs *pTfs;
TdThread thread;
bool stop;
TdThreadMutex createLock;
} SVnodeMgmt;
typedef struct {
@ -69,6 +71,7 @@ typedef struct {
STaosQueue *pQueryQ;
STaosQueue *pStreamQ;
STaosQueue *pFetchQ;
STaosQueue *pMultiMgmQ;
} SVnodeObj;
typedef struct {
@ -125,6 +128,7 @@ int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
#ifdef __cplusplus
}

5
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -377,11 +377,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
goto _OVER;
}
taosThreadMutexLock(&pMgmt->createLock);
code = vmWriteVnodeListToFile(pMgmt);
if (code != 0) {
code = terrno != 0 ? terrno : code;
taosThreadMutexUnlock(&pMgmt->createLock);
goto _OVER;
}
taosThreadMutexUnlock(&pMgmt->createLock);
_OVER:
if (code != 0) {
@ -985,7 +988,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMultiMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

4
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -439,6 +439,8 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
dInfo("start to close all vnodes");
tSingleWorkerCleanup(&pMgmt->mgmtWorker);
dInfo("vnodes mgmt worker is stopped");
tSingleWorkerCleanup(&pMgmt->mgmtMultiWorker);
dInfo("vnodes multiple mgmt worker is stopped");
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
@ -506,6 +508,7 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
vmStopWorker(pMgmt);
vnodeCleanup();
taosThreadRwlockDestroy(&pMgmt->lock);
taosThreadMutexDestroy(&pMgmt->createLock);
taosMemoryFree(pMgmt);
}
@ -580,6 +583,7 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize;
pMgmt->msgCb.mgmt = pMgmt;
taosThreadRwlockInit(&pMgmt->lock, NULL);
taosThreadMutexInit(&pMgmt->createLock, NULL);
pMgmt->pTfs = pInput->pTfs;
if (pMgmt->pTfs == NULL) {

46
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -28,6 +28,31 @@ static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) {
tmsgSendRsp(&rsp);
}
static void vmProcessMultiMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SVnodeMgmt *pMgmt = pInfo->ahandle;
int32_t code = -1;
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, get from vnode-multi-mgmt queue", pMsg);
switch (pMsg->msgType) {
case TDMT_DND_CREATE_VNODE:
code = vmProcessCreateVnodeReq(pMgmt, pMsg);
break;
}
if (IsReq(pMsg)) {
if (code != 0) {
if (terrno != 0) code = terrno;
dGError("msg:%p, failed to process since %s, type:%s", pMsg, tstrerror(code), TMSG_INFO(pMsg->msgType));
}
vmSendRsp(pMsg, code);
}
dGTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SVnodeMgmt *pMgmt = pInfo->ahandle;
int32_t code = -1;
@ -271,6 +296,13 @@ int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsg
int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, STREAM_QUEUE); }
int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-multi-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg);
return 0;
}
int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-mgmt queue", pMsg);
@ -415,6 +447,20 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) return -1;
int32_t threadNum = 0;
if (tsNumOfCores == 1) {
threadNum = 2;
} else {
threadNum = tsNumOfCores;
}
SSingleWorkerCfg multiMgmtCfg = {.min = threadNum,
.max = threadNum,
.name = "vnode-multi-mgmt",
.fp = (FItem)vmProcessMultiMgmtQueue,
.param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg) != 0) return -1;
dDebug("vnode workers are initialized");
return 0;
}

5
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -18,6 +18,7 @@
#include "audit.h"
#include "libs/function/tudf.h"
#include "tgrant.h"
#include "tcompare.h"
#define DM_INIT_AUDIT() \
do { \
@ -163,6 +164,7 @@ int32_t dmInit() {
if (dmInitMonitor() != 0) return -1;
if (dmInitAudit() != 0) return -1;
if (dmInitDnode(dmInstance()) != 0) return -1;
if (InitRegexCache() != 0) return -1;
#if defined(USE_S3)
if (s3Begin() != 0) return -1;
#endif
@ -192,6 +194,7 @@ void dmCleanup() {
udfStopUdfd();
taosStopCacheRefreshWorker();
dmDiskClose();
DestroyRegexCache();
#if defined(USE_S3)
s3End();
@ -310,7 +313,7 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) {
dmReleaseWrapper(pWrapper);
taosThreadMutexUnlock(&pDnode->mutex);
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
return -1;

4
source/dnode/mnode/impl/src/mndDef.c
View File

@ -195,7 +195,9 @@ void *freeStreamTasks(SArray *pTaskLevel) {
taosArrayDestroy(pLevel);
}
return taosArrayDestroy(pTaskLevel);
taosArrayDestroy(pTaskLevel);
return NULL;
}
void tFreeStreamObj(SStreamObj *pStream) {

3
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -978,7 +978,8 @@ void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpo
}
void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
pInfo->pTaskList = taosArrayDestroy(pInfo->pTaskList);
taosArrayDestroy(pInfo->pTaskList);
pInfo->pTaskList = NULL;
}
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) {

2
source/dnode/vnode/inc/vnode.h
View File

@ -167,7 +167,7 @@ void tsdbReaderClose2(STsdbReader *pReader);
int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext);
int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA);
void tsdbReleaseDataBlock2(STsdbReader *pReader);
SSDataBlock *tsdbRetrieveDataBlock2(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList);
int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond);
int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle);

6
source/dnode/vnode/src/inc/tsdb.h
View File

@ -909,9 +909,9 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree);
bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree *pMTree);
SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols);
void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void * destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo);
void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
// tsdbCache ==============================================================================================
typedef enum {

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -54,7 +54,6 @@ extern "C" {
#endif
typedef struct SVnodeInfo SVnodeInfo;
typedef struct SMeta SMeta;
typedef struct SSma SSma;
typedef struct STsdb STsdb;
typedef struct STQ STQ;
@ -153,7 +152,6 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode);
void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive);
// meta
typedef struct SMCtbCursor SMCtbCursor;
typedef struct SMStbCursor SMStbCursor;
typedef struct STbUidStore STbUidStore;

9
source/dnode/vnode/src/tq/tqSink.c
View File

@ -250,7 +250,8 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
}
tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
tagArray = taosArrayDestroy(tagArray);
taosArrayDestroy(tagArray);
tagArray = NULL;
if (pCreateTbReq->ctb.pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq);
code = TSDB_CODE_OUT_OF_MEMORY;
@ -514,7 +515,8 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
pTableData->aRowP = taosArrayInit(numOfRows, sizeof(SRow*));
if (pTableData->aRowP == NULL || pVals == NULL) {
pTableData->aRowP = taosArrayDestroy(pTableData->aRowP);
taosArrayDestroy(pTableData->aRowP);
pTableData->aRowP = NULL;
taosArrayDestroy(pVals);
code = TSDB_CODE_OUT_OF_MEMORY;
tqError("s-task:%s failed to prepare write stream res blocks, code:%s", id, tstrerror(code));
@ -539,7 +541,8 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
if (ts < earlyTs) {
tqError("s-task:%s ts:%" PRId64 " of generated results out of valid time range %" PRId64 " , discarded", id,
ts, earlyTs);
pTableData->aRowP = taosArrayDestroy(pTableData->aRowP);
taosArrayDestroy(pTableData->aRowP);
pTableData->aRowP = NULL;
taosArrayDestroy(pVals);
return TSDB_CODE_SUCCESS;
}

12
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -54,19 +54,23 @@ static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) {
if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
if (pRsp->blockData != NULL) {
pRsp->blockData = taosArrayDestroy(pRsp->blockData);
taosArrayDestroy(pRsp->blockData);
pRsp->blockData = NULL;
}
if (pRsp->blockDataLen != NULL) {
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);
taosArrayDestroy(pRsp->blockDataLen);
pRsp->blockDataLen = NULL;
}
if (pRsp->blockTbName != NULL) {
pRsp->blockTbName = taosArrayDestroy(pRsp->blockTbName);
taosArrayDestroy(pRsp->blockTbName);
pRsp->blockTbName = NULL;
}
if (pRsp->blockSchema != NULL) {
pRsp->blockSchema = taosArrayDestroy(pRsp->blockSchema);
taosArrayDestroy(pRsp->blockSchema);
pRsp->blockSchema = NULL;
}
return -1;
}

10
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -2579,7 +2579,7 @@ typedef struct {
static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) {
int32_t code = 0;
pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, NULL);
destroySttBlockReader(pr->pLDataIterArray, NULL);
pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
SMergeTreeConf conf = {
@ -3187,7 +3187,8 @@ static void clearLastFileSet(SFSNextRowIter *state) {
int32_t iter = 0;
while ((pe = tSimpleHashIterate(state->pr->pTableMap, pe, &iter)) != NULL) {
STableLoadInfo *pInfo = *(STableLoadInfo **)pe;
pInfo->pTombData = taosArrayDestroy(pInfo->pTombData);
taosArrayDestroy(pInfo->pTombData);
pInfo->pTombData = NULL;
}
}
}
@ -3211,7 +3212,10 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
pIter->pMemDelData = NULL;
loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer);
code = loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
pIter->idx = (SBlockIdx){.suid = suid, .uid = uid};

5
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -203,7 +203,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf
pReader->pTableList = pTableIdList;
pReader->numOfTables = numOfTables;
pReader->lastTs = INT64_MIN;
pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, NULL);
destroySttBlockReader(pReader->pLDataIterArray, NULL);
pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
return TSDB_CODE_SUCCESS;
@ -311,7 +311,8 @@ void* tsdbCacherowsReaderClose(void* pReader) {
int32_t iter = 0;
while ((pe = tSimpleHashIterate(p->pTableMap, pe, &iter)) != NULL) {
STableLoadInfo* pInfo = *(STableLoadInfo**)pe;
pInfo->pTombData = taosArrayDestroy(pInfo->pTombData);
taosArrayDestroy(pInfo->pTombData);
pInfo->pTombData = NULL;
}
tSimpleHashCleanup(p->pTableMap);

35
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -23,11 +23,12 @@
static void tLDataIterClose2(SLDataIter *pIter);
// SLDataIter =================================================
SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) {
int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo) {
*pInfo = NULL;
SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo));
if (pLoadInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
pLoadInfo->blockData[0].sttBlockIndex = -1;
@ -37,26 +38,29 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList,
int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data);
if (code) {
terrno = code;
taosMemoryFreeClear(pLoadInfo);
return code;
}
code = tBlockDataCreate(&pLoadInfo->blockData[1].data);
if (code) {
terrno = code;
taosMemoryFreeClear(pLoadInfo);
return code;
}
pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk));
if (pLoadInfo->aSttBlk == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFreeClear(pLoadInfo);
return NULL;
return code;
}
pLoadInfo->pSchema = pSchema;
pLoadInfo->colIds = colList;
pLoadInfo->numOfCols = numOfCols;
return pLoadInfo;
*pInfo = pLoadInfo;
return code;
}
static void freeItem(void* pValue) {
@ -66,9 +70,9 @@ static void freeItem(void* pValue) {
}
}
void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
if (pLoadInfo == NULL) {
return NULL;
return;
}
pLoadInfo->currentLoadBlockIndex = 1;
@ -94,7 +98,6 @@ void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
taosArrayDestroy(pLoadInfo->aSttBlk);
taosMemoryFree(pLoadInfo);
return NULL;
}
void destroyLDataIter(SLDataIter *pIter) {
@ -103,9 +106,9 @@ void destroyLDataIter(SLDataIter *pIter) {
taosMemoryFree(pIter);
}
void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) {
void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) {
if (pLDataIterArray == NULL) {
return NULL;
return;
}
int32_t numOfLevel = taosArrayGetSize(pLDataIterArray);
@ -132,7 +135,6 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa
}
taosArrayDestroy(pLDataIterArray);
return NULL;
}
// choose the unpinned slot to load next data block
@ -914,9 +916,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF
}
if (pLoadInfo == NULL) {
pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols);
if (pLoadInfo == NULL) {
code = terrno;
code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pLoadInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
}

761
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

File diff suppressed because it is too large Load Diff

136
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -112,22 +112,28 @@ void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {
taosArrayDestroy(pBuf->pData);
}
void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) {
int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) {
*pInfo = NULL;
int32_t bucketIndex = index / pBuf->numPerBucket;
char** pBucket = taosArrayGet(pBuf->pData, bucketIndex);
return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo);
}
STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) {
STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (p == NULL || *p == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
int32_t size = tSimpleHashGetSize(pTableMap);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);
return NULL;
if (pBucket == NULL) {
return TSDB_CODE_FAILED;
}
return *p;
*pInfo = (STableBlockScanInfo*)((*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo));
return TSDB_CODE_SUCCESS;
}
int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) {
*pInfo = *(STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (pInfo == NULL) {
int32_t size = tSimpleHashGetSize(pTableMap);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);
return TSDB_CODE_INVALID_PARA;
}
return TSDB_CODE_SUCCESS;
}
int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) {
@ -146,17 +152,17 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
}
case TSDB_DATA_TYPE_INT:{
int32_t min = INT32_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_SMALLINT:{
int16_t min = INT16_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_TINYINT:{
int8_t min = INT8_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_UTINYINT:
@ -245,20 +251,27 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH
initLastProcKey(pScanInfo, pReader);
pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
int32_t code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid,
pScanInfo->lastProcKey.ts, pReader->idStr);
return TSDB_CODE_SUCCESS;
return code;
}
// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model
SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables) {
int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) {
int32_t code = 0;
*pHashObj = NULL;
// allocate buffer in order to load data blocks from file
// todo use simple hash instead, optimize the memory consumption
SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (pTableMap == NULL) {
return NULL;
return terrno;
}
int64_t st = taosGetTimestampUs();
@ -267,7 +280,7 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
if (pUidList->tableUidList == NULL) {
tSimpleHashCleanup(pTableMap);
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
pUidList->currentIndex = 0;
@ -275,8 +288,16 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
for (int32_t j = 0; j < numOfTables; ++j) {
pUidList->tableUidList[j] = idList[j].uid;
STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j);
initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);
STableBlockScanInfo* pScanInfo = NULL;
code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo);
if (code != TSDB_CODE_SUCCESS) {
break;
}
code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);
if (code != TSDB_CODE_SUCCESS) {
break;
}
}
taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc);
@ -286,7 +307,8 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
(sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList,
pTsdbReader->idStr);
return pTableMap;
*pHashObj = pTableMap;
return code;
}
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {
@ -308,7 +330,8 @@ void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {
pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter);
}
pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline);
taosArrayDestroy(pInfo->delSkyline);
pInfo->delSkyline = NULL;
pInfo->lastProcKey.ts = ts;
// todo check the nextProcKey info
pInfo->sttKeyInfo.nextProcKey.ts = ts + step;
@ -329,11 +352,16 @@ void clearBlockScanInfo(STableBlockScanInfo* p) {
p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter);
}
p->delSkyline = taosArrayDestroy(p->delSkyline);
p->pBlockList = taosArrayDestroy(p->pBlockList);
p->pBlockIdxList = taosArrayDestroy(p->pBlockIdxList);
p->pMemDelData = taosArrayDestroy(p->pMemDelData);
p->pFileDelData = taosArrayDestroy(p->pFileDelData);
taosArrayDestroy(p->delSkyline);
p->delSkyline = NULL;
taosArrayDestroy(p->pBlockList);
p->pBlockList = NULL;
taosArrayDestroy(p->pBlockIdxList);
p->pBlockIdxList = NULL;
taosArrayDestroy(p->pMemDelData);
p->pMemDelData = NULL;
taosArrayDestroy(p->pFileDelData);
p->pFileDelData = NULL;
clearRowKey(&p->lastProcKey);
clearRowKey(&p->sttRange.skey);
@ -385,11 +413,13 @@ void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray
pIter->pBrinBlockList = pList;
}
SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {
*pRecord = NULL;
if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) {
pIter->blockIndex += 1;
if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) {
return NULL;
return TSDB_CODE_FAILED;
}
pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);
@ -398,7 +428,7 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));
return NULL;
return TSDB_CODE_FAILED;
}
pIter->recordIndex = -1;
@ -406,7 +436,9 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
pIter->recordIndex += 1;
tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record);
return &pIter->record;
*pRecord = &pIter->record;
return TSDB_CODE_SUCCESS;
}
void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); }
@ -579,7 +611,8 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
}
taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList);
pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList);
taosArrayDestroy(pTableScanInfo->pBlockList);
pTableScanInfo->pBlockList = NULL;
int64_t et = taosGetTimestampUs();
tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s",
@ -624,7 +657,8 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
for (int32_t i = 0; i < numOfTables; ++i) {
STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i);
pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList);
taosArrayDestroy(pTableScanInfo->pBlockList);
pTableScanInfo->pBlockList = NULL;
}
int64_t et = taosGetTimestampUs();
@ -662,7 +696,13 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
STombRecord record = {0};
uint64_t uid = pReader->status.uidList.tableUidList[*j];
STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr);
STableBlockScanInfo* pScanInfo = NULL;
code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
}
@ -694,7 +734,11 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
}
uid = pReader->status.uidList.tableUidList[*j];
pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr);
code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
}
@ -798,9 +842,12 @@ int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileRead
return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false);
}
void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {
int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {
if (*ppMemDelData == NULL) {
*ppMemDelData = taosArrayInit(4, sizeof(SDelData));
if (*ppMemDelData == NULL) {
return TSDB_CODE_SUCCESS;
}
}
SArray* pMemDelData = *ppMemDelData;
@ -828,6 +875,8 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT
p = p->pNext;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
@ -952,6 +1001,7 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet)
int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf,
const char* pstr) {
int32_t numOfRows = 0;
int32_t code = 0;
// no data exists, go to end
int32_t numOfLevels = pFileSet->lvlArr->size;
@ -976,7 +1026,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
conf.file[0] = *pSttLevel->fobjArr->data[i]->f;
const char* pName = pSttLevel->fobjArr->data[i]->fname;
int32_t code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);
code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr);
continue;
@ -984,8 +1034,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
}
if (pIter->pBlockLoadInfo == NULL) {
pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols);
if (pIter->pBlockLoadInfo == NULL) {
code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to create block load info, code: out of memory, %s", pstr);
continue;
}
@ -993,7 +1043,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
// load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file
TStatisBlkArray* pStatisBlkArray = NULL;
int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);
code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr);
continue;

41
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -315,32 +315,31 @@ typedef struct SBrinRecordIter {
} SBrinRecordIter;
int32_t uidComparFunc(const void* p1, const void* p2);
int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id);
STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id);
SSHashObj* createDataBlockScanInfo(STsdbReader* pReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables);
int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,
STsdbReader* pReader);
void clearBlockScanInfo(STableBlockScanInfo* p);
void destroyAllBlockScanInfo(SSHashObj* pTableMap);
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step);
void cleanupInfoForNextFileset(SSHashObj* pTableMap);
int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables);
void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf);
void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index);
int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj);
int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,
STsdbReader* pReader);
void clearBlockScanInfo(STableBlockScanInfo* p);
void destroyAllBlockScanInfo(SSHashObj* pTableMap);
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step);
void cleanupInfoForNextFileset(SSHashObj* pTableMap);
int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables);
void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf);
int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pRes);
// brin records iterator
void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList);
SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter);
void clearBrinBlockIter(SBrinRecordIter* pIter);
void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList);
int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord);
void clearBrinBlockIter(SBrinRecordIter* pIter);
// initialize block iterator API
int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList);
bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr);
// load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time)
void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver);
int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver);
int32_t loadDataFileTombDataForAll(STsdbReader* pReader);
int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo);
int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
@ -358,10 +357,10 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2);
int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc);
void clearRowKey(SRowKey* pKey);
bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp);
void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk);
void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree);
void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk);
bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp);
int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk);
void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree);
void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk);
typedef struct {
SArray* pTombData;

4
source/libs/audit/src/auditMain.c
View File

@ -32,8 +32,8 @@ char* tsAuditBatchUri = "/audit-batch";
int32_t auditInit(const SAuditCfg *pCfg) {
tsAudit.cfg = *pCfg;
tsAudit.records = taosArrayInit(0, sizeof(SAuditRecord *));
taosThreadMutexInit(&tsAudit.lock, NULL);
return 0;
if(tsAudit.records == NULL) return TSDB_CODE_OUT_OF_MEMORY;
return taosThreadMutexInit(&tsAudit.lock, NULL);
}
static FORCE_INLINE void auditDeleteRecord(SAuditRecord * record) {

2
source/libs/executor/inc/executorInt.h
View File

@ -929,7 +929,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i
SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore,
SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr,
SStorageAPI* pApi, int32_t tsIndex);
void initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic);
void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins);
void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);

1
source/libs/executor/inc/streamexecutorInt.h
View File

@ -20,6 +20,7 @@ extern "C" {
#endif
#include "executorInt.h"
#include "tutil.h"
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);

6
source/libs/executor/src/executil.c
View File

@ -96,7 +96,8 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
pGroupResInfo->freeItem = false;
pGroupResInfo->pRows = NULL;
} else {
pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows);
taosArrayDestroy(pGroupResInfo->pRows);
pGroupResInfo->pRows = NULL;
}
pGroupResInfo->index = 0;
}
@ -2102,7 +2103,8 @@ void* tableListDestroy(STableListInfo* pTableListInfo) {
return NULL;
}
pTableListInfo->pTableList = taosArrayDestroy(pTableListInfo->pTableList);
taosArrayDestroy(pTableListInfo->pTableList);
pTableListInfo->pTableList = NULL;
taosMemoryFreeClear(pTableListInfo->groupOffset);
taosHashCleanup(pTableListInfo->map);

8
source/libs/executor/src/executor.c
View File

@ -479,12 +479,14 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
}
int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
int32_t* tversion, int32_t idx) {
int32_t* tversion, int32_t idx, bool* tbGet) {
*tbGet = false;
ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL);
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
if (taosArrayGetSize(pTaskInfo->schemaInfos) <= idx) {
return -1;
return TSDB_CODE_SUCCESS;
}
SSchemaInfo* pSchemaInfo = taosArrayGet(pTaskInfo->schemaInfos, idx);
@ -502,6 +504,8 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
tableName[0] = 0;
}
*tbGet = true;
return TSDB_CODE_SUCCESS;
}

18
source/libs/executor/src/groupoperator.c
View File

@ -17,6 +17,7 @@
#include "function.h"
#include "os.h"
#include "tname.h"
#include "tutil.h"
#include "tdatablock.h"
#include "tmsg.h"
@ -1357,11 +1358,13 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
taosMemoryFreeClear(param);
}
void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) {
int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI;
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
return;
return code;
}
SStreamScanInfo* pScanInfo = downstream->info;
@ -1369,8 +1372,9 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup
pScanInfo->pPartScalarSup = pExpr;
pScanInfo->pPartTbnameSup = pTbnameExpr;
if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen);
code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
}
return code;
}
SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) {
@ -1421,6 +1425,7 @@ void freePartItem(void* ptr) {
SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -1511,14 +1516,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup);
code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
return pOperator;
_error:
pTaskInfo->code = code;
destroyStreamPartitionOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator);
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}

88
source/libs/executor/src/scanoperator.c
View File

@ -388,9 +388,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1;
SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
if (p == NULL) {
return terrno;
SSDataBlock* p = NULL;
int32_t code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, &p, NULL);
if (p == NULL || code != TSDB_CODE_SUCCESS) {
return code;
}
ASSERT(p == pBlock);
@ -1351,7 +1352,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
}
if (hasNext) {
/*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL);
SSDataBlock* p = NULL;
code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, &p, NULL);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
}
@ -2477,27 +2483,54 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl
}
}
int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) {
int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo);
len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup);
*pBuff = taosMemoryCalloc(1, len);
void* buf = *pBuff;
encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup);
pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo);
return len;
int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff, int32_t* pLen) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t len = 0;
code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len);
TSDB_CHECK_CODE(code, lino, _end);
len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup);
*pBuff = taosMemoryCalloc(1, len);
if (!(*pBuff)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
void* buf = *pBuff;
encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup);
int32_t tmp = 0;
code = pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo, &tmp);
TSDB_CHECK_CODE(code, lino, _end);
*pLen = len;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (!pInfo->pState) {
return;
}
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
void* pBuf = NULL;
int32_t len = streamScanOperatorEncode(pInfo, &pBuf);
int32_t len = 0;
code = streamScanOperatorEncode(pInfo, &pBuf, &len);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
taosMemoryFree(pBuf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
}
// other properties are recovered from the execution plan
@ -2929,8 +2962,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL);
if (pBlock == NULL) {
SSDataBlock* pBlock = NULL;
code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, &pBlock, NULL);
if (pBlock == NULL || code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
@ -3094,17 +3128,35 @@ static void destroyStreamScanOperatorInfo(void* param) {
}
void streamScanReleaseState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamScanInfo* pInfo = pOperator->info;
void* pBuff = NULL;
if (!pInfo->pState) {
return;
}
if (!pInfo->pUpdateInfo) {
return;
}
int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo);
void* pBuff = taosMemoryCalloc(1, len);
pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo);
int32_t len = 0;
code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len);
TSDB_CHECK_CODE(code, lino, _end);
pBuff = taosMemoryCalloc(1, len);
if (!pBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
int32_t tmp = 0;
code = pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo, &tmp);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosMemoryFree(pBuff);
}

90
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -68,9 +68,7 @@ void destroyStreamCountAggOperatorInfo(void* param) {
taosMemoryFreeClear(param);
}
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) {
return pAggSup->windowCount != pAggSup->windowSliding;
}
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; }
void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
SBuffInfo* pBuffInfo) {
@ -89,13 +87,14 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
ASSERT(pBuffInfo->pCur);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
(void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
}
} else {
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount);
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) {
@ -107,15 +106,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
pBuffInfo->rebuildWindow = true;
}
} else {
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(
pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size);
}
if (code == TSDB_CODE_SUCCESS) {
pCurWin->winInfo.isOutput = true;
}
pCurWin->pWindowCount=
(COUNT_TYPE*) ((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE)));
pCurWin->pWindowCount =
(COUNT_TYPE*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE)));
if (*pCurWin->pWindowCount == pAggSup->windowCount) {
pBuffInfo->rebuildWindow = true;
@ -133,7 +133,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI
int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool* pRebuild) {
SSessionKey sWinKey = pWinInfo->winInfo.sessionWin;
int32_t num = 0;
int32_t num = 0;
for (int32_t i = start; i < rows; i++) {
if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) {
num++;
@ -181,7 +181,7 @@ void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, ESt
pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey);
}
SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0);
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
pAggSup->stateStore.streamStateFreeCur(pCur);
return;
@ -239,7 +239,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2);
SStreamStateCur* pCur = NULL;
COUNT_TYPE slidingRows = 0;
COUNT_TYPE slidingRows = 0;
for (int32_t i = 0; i < rows;) {
if (pInfo->ignoreExpiredData &&
@ -269,7 +269,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
}
getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range);
range.win.skey = TMIN(startTsCols[i], range.win.skey);
range.win.ekey = TMAX(startTsCols[rows-1], range.win.ekey);
range.win.ekey = TMAX(startTsCols[rows - 1], range.win.ekey);
uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL);
break;
@ -289,8 +289,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
code = saveResult(curWin.winInfo, pStUpdated);
if (code != TSDB_CODE_SUCCESS) {
qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo),
tstrerror(code));
qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
break;
}
}
@ -397,13 +396,14 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
int32_t mapSize = 0;
buf = taosDecodeFixedI32(buf, &mapSize);
for (int32_t i = 0; i < mapSize; i++) {
SSessionKey key = {0};
SSessionKey key = {0};
SCountWindowInfo curWin = {0};
buf = decodeSSessionKey(buf, &key);
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo,
sizeof(SResultWindowInfo));
}
// 2.twAggSup
@ -441,9 +441,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
uint64_t* gpDatas = (uint64_t*)pGroupCol->pData;
SRowBuffPos* pPos = NULL;
int32_t size = 0;
int32_t size = 0;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]};
SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]};
SStreamStateCur* pCur = NULL;
if (isSlidingCountWindow(pAggSup)) {
pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount);
@ -452,7 +452,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
}
while (1) {
SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size);
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void**)&pPos, &size);
if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) {
pAggSup->stateStore.streamStateFreeCur(pCur);
break;
@ -482,7 +482,7 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr
SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]};
while (1) {
SSessionKey curWin = {0};
int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin);
int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin);
if (code == TSDB_CODE_FAILED) {
break;
}
@ -511,11 +511,11 @@ void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHa
}
static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pOperator->exprSupp;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pSup = &pOperator->exprSupp;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
@ -635,7 +635,7 @@ void streamCountReloadState(SOperatorInfo* pOperator) {
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME,
strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size);
TSKEY ts = *(TSKEY*)pBuf;
TSKEY ts = *(TSKEY*)pBuf;
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
taosMemoryFree(pBuf);
@ -650,11 +650,13 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) {
SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode;
int32_t numOfCols = 0;
int32_t code = TSDB_CODE_OUT_OF_MEMORY;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pOperator->pTaskInfo = pTaskInfo;
@ -664,18 +666,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
}
SExprSupp* pExpSup = &pOperator->exprSupp;
SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
pInfo->twAggSup = (STimeWindowAggSupp){
.waterMark = pCountNode->window.watermark,
@ -686,12 +684,11 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
};
pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId;
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0,
pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle,
&pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState,
sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
TSDB_CHECK_CODE(code, lino, _error);
pInfo->streamAggSup.windowCount = pCountNode->windowCount;
pInfo->streamAggSup.windowSliding = pCountNode->windowSliding;
@ -709,7 +706,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) {
goto _error;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
@ -735,8 +733,12 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
}
return pOperator;
@ -747,6 +749,6 @@ _error:
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}

112
source/libs/executor/src/streameventwindowoperator.c
View File

@ -28,9 +28,9 @@
#include "tlog.h"
#include "ttime.h"
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
typedef struct SEventWinfowFlag {
bool startFlag;
@ -86,10 +86,11 @@ void destroyStreamEventOperatorInfo(void* param) {
void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) {
char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize);
pWinInfo->pWinFlag = (SEventWinfowFlag*) pFlagInfo;
pWinInfo->pWinFlag = (SEventWinfowFlag*)pFlagInfo;
}
void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, SEventWindowInfo* pWinInfo) {
void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos,
SEventWindowInfo* pWinInfo) {
pWinInfo->winInfo.sessionWin = *pKey;
pWinInfo->winInfo.pStatePos = pPos;
setEventWindowFlag(pAggSup, pWinInfo);
@ -122,22 +123,23 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts;
pCurWin->winInfo.sessionWin.win.ekey = ts;
SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin);
SSessionKey leftWinKey = {.groupId = groupId};
void* pVal = NULL;
int32_t len = 0;
SStreamStateCur* pCur =
pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin);
SSessionKey leftWinKey = {.groupId = groupId};
void* pVal = NULL;
int32_t len = 0;
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win) ) {
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) {
bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0);
setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin);
if(inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag) ) {
if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) {
pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin);
goto _end;
}
}
pAggSup->stateStore.streamStateFreeCur(pCur);
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
SSessionKey rightWinKey = {.groupId = groupId};
SSessionKey rightWinKey = {.groupId = groupId};
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len);
bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) {
@ -149,7 +151,7 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
}
}
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
pCurWin->pWinFlag->startFlag = start;
@ -164,7 +166,7 @@ _end:
if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pNextWinKey);
}
if(pCurWin->winInfo.pStatePos->needFree) {
if (pCurWin->winInfo.pStatePos->needFree) {
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);
}
pAggSup->stateStore.streamStateFreeCur(pCur);
@ -173,14 +175,14 @@ _end:
}
int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey,
TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows,
SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) {
TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start,
SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) {
*pRebuild = false;
if (!pWinInfo->pWinFlag->startFlag && !(starts[start]) ) {
if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) {
return 1;
}
TSKEY maxTs = INT64_MAX;
TSKEY maxTs = INT64_MAX;
STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win;
if (pWinInfo->pWinFlag->endFlag) {
maxTs = pWin->ekey + 1;
@ -219,7 +221,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));
if (ends[i]) {
if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey ) {
if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) {
*pRebuild = true;
}
return i + 1 - start;
@ -230,26 +232,28 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool addGap) {
SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
int32_t winNum = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
int32_t winNum = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
while (1) {
if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) {
break;
}
SEventWindowInfo nextWinInfo = {0};
getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo);
if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) {
if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) ||
!inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) {
releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);
break;
}
setEventWindowFlag(pAggSup, &nextWinInfo);
compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, pStDeleted, false);
compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated,
pStDeleted, false);
pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag;
winNum++;
}
@ -275,7 +279,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t winRows = 0;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SColumnInfoData* pColStart = NULL;
SColumnInfoData* pColEnd = NULL;
SColumnInfoData* pColEnd = NULL;
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
@ -290,7 +294,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
return;
}
SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock};
SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pStartCondInfo, &paramStart);
if (code != TSDB_CODE_SUCCESS) {
qError("set data from start slotId error.");
@ -299,7 +304,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t statusStart = 0;
filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart);
SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock};
SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pEndCondInfo, &paramEnd);
if (code != TSDB_CODE_SUCCESS) {
qError("set data from end slotId error.");
@ -320,20 +326,23 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t winIndex = 0;
bool allEqual = true;
SEventWindowInfo curWin = {0};
SSessionKey nextWinKey = {0};
setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, &nextWinKey);
SSessionKey nextWinKey = {0};
setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin,
&nextWinKey);
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
bool rebuild = false;
winRows = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i,
pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild);
winRows =
updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild);
ASSERT(winRows >= 1);
if (rebuild) {
uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin);
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) {
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) &&
!isWindowIncomplete(&curWin)) {
saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
}
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
@ -603,7 +612,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) {
char* pBuff = taosMemoryCalloc(1, resSize);
memcpy(pBuff, pInfo->historyWins->pData, winSize);
memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
qDebug("===stream=== event window operator relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins));
qDebug("===stream=== event window operator relase state. save result count:%d",
(int32_t)taosArrayGetSize(pInfo->historyWins));
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME,
strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize);
pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);
@ -647,7 +657,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,
pSeKeyBuf[i].groupId, i);
getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo);
//event window has been deleted
// event window has been deleted
if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) {
continue;
}
@ -659,7 +669,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
curInfo.winInfo.sessionWin.groupId);
if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {
saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
}
@ -693,6 +703,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode;
int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -783,20 +794,18 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
return pOperator;
@ -804,5 +813,6 @@ _error:
destroyStreamEventOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}

3
source/libs/executor/src/streamfilloperator.c
View File

@ -129,7 +129,8 @@ static void destroyStreamFillOperatorInfo(void* param) {
pInfo->pRes = blockDataDestroy(pInfo->pRes);
pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock);
pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList);
taosArrayDestroy(pInfo->matchInfo.pList);
pInfo->matchInfo.pList = NULL;
taosMemoryFree(pInfo);
}

195
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -28,14 +28,18 @@
#include "tlog.h"
#include "ttime.h"
#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
#define IS_NORMAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
#define IS_NORMAL_INTERVAL_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_SESSION_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
#define DEAULT_DELETE_MARK INT64_MAX
#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState"
@ -263,14 +267,16 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
if (chIds) {
int32_t childId = getChildIndex(pBlock);
if (pInvalidWins) {
qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId);
qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts,
winRes.groupId, childId);
taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0);
}
SArray* chArray = *(void**)chIds;
int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);
if (index != -1) {
qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, winGpId, childId);
qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey,
winGpId, childId);
getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
continue;
}
@ -390,7 +396,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
}
void destroyFlusedPos(void* pRes) {
SRowBuffPos* pPos = (SRowBuffPos*) pRes;
SRowBuffPos* pPos = (SRowBuffPos*)pRes;
if (!pPos->needFree && !pPos->pRowBuff) {
taosMemoryFreeClear(pPos->pKey);
taosMemoryFree(pPos);
@ -406,7 +412,8 @@ void clearGroupResInfo(SGroupResInfo* pGroupResInfo) {
}
pGroupResInfo->freeItem = false;
}
pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows);
taosArrayDestroy(pGroupResInfo->pRows);
pGroupResInfo->pRows = NULL;
pGroupResInfo->index = 0;
}
@ -487,8 +494,11 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt
pScanInfo->windowSup.parentType = type;
pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;
if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo =
pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen);
int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate,
pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at %d since %s", __func__, __LINE__, tstrerror(code));
}
}
pScanInfo->interval = pInfo->interval;
@ -623,17 +633,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
blockDataUpdateTsWindow(pBlock, 0);
}
static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins,
int32_t numOfCh, SOperatorInfo* pOperator) {
static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval,
SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* tsData = (TSKEY*)pStartCol->pData;
SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
TSKEY* tsEndData = (TSKEY*)pEndCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
int32_t chId = getChildIndex(pBlock);
bool res = false;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* tsData = (TSKEY*)pStartCol->pData;
SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
TSKEY* tsEndData = (TSKEY*)pEndCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
int32_t chId = getChildIndex(pBlock);
bool res = false;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
TSKEY winTs = tsData[i];
while (winTs <= tsEndData[i]) {
@ -649,7 +659,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
// pull data is over
taosArrayDestroy(chArray);
taosHashRemove(pMap, &winRes, sizeof(SWinKey));
res =true;
res = true;
qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts);
void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey));
@ -662,7 +672,8 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
.calWin.skey = nextWin.skey,
.calWin.ekey = nextWin.skey};
// add pull data request
qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", winRes.ts, winRes.groupId, numOfCh);
qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d",
winRes.ts, winRes.groupId, numOfCh);
if (IS_MID_INTERVAL_OP(pOperator)) {
SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;
taosArrayPush(pInfo->pMidPullDatas, &winRes);
@ -670,7 +681,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
taosArrayPush(pInfo->pDelWins, &winRes);
addPullWindow(pMap, &winRes, numOfCh);
if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0);
tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);
}
qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh);
}
@ -689,7 +700,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i
for (int32_t i = 0; i < size; i++) {
SWinKey* winKey = taosArrayGet(wins, i);
STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
if (!chIds) {
SPullWindowInfo pull = {
.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
@ -697,7 +708,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild);
if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,winKey, sizeof(SWinKey), NULL, 0);
tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0);
}
qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild);
}
@ -750,7 +761,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat
if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = groupId;
void* tbname = NULL;
if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, false) < 0) {
if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,
false) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -828,9 +840,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN
return startPos;
}
bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) {
return pInfo->primaryPkIndex != -1;
}
bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; }
static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId,
SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) {
@ -851,23 +861,22 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData;
void* pPkVal = NULL;
int32_t pkLen = 0;
void* pPkVal = NULL;
int32_t pkLen = 0;
SColumnInfoData* pPkColDataInfo = NULL;
if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
}
if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) {
qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64
",maxKey %" PRId64,
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64,
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex);
// timestamp of the data is incorrect
if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {
qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
}
}
@ -915,7 +924,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild);
if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0);
tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);
}
}
} else {
@ -961,7 +970,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
};
if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) {
tSimpleHashPut(pDeletedMap,&key, sizeof(SWinKey), NULL, 0);
tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0);
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
@ -1364,7 +1373,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
pInfo->binfo.pRes->info.type = pBlock->info.type;
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL;
doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap);
if (IS_FINAL_INTERVAL_OP(pOperator)) {
@ -1398,7 +1407,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);
continue;
} else if (pBlock->info.type == STREAM_RETRIEVE) {
if(!IS_FINAL_INTERVAL_OP(pOperator)) {
if (!IS_FINAL_INTERVAL_OP(pOperator)) {
pInfo->recvRetrive = true;
copyDataBlock(pInfo->pMidRetriveRes, pBlock);
pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;
@ -1524,8 +1533,8 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&
pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
int32_t size = 0;
void* pBuf = NULL;
int32_t size = 0;
void* pBuf = NULL;
int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,
strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size);
if (code == 0) {
@ -1599,7 +1608,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pInfo->pState, &pTaskInfo->storageAPI.functionStore);
pInfo->pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1754,27 +1763,37 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
}
}
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) {
int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
pScanInfo->tsColIndex = tsColIndex;
}
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic);
return;
code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic);
return code;
}
SStreamScanInfo* pScanInfo = downstream->info;
pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
pScanInfo->pState = pAggSup->pState;
if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,
pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen);
code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,
pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen,
&pScanInfo->pUpdateInfo);
TSDB_CHECK_CODE(code, lino, _end);
}
pScanInfo->twAggSup = *pTwSup;
pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;
pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static TSKEY sesionTs(void* pKey) {
@ -2147,8 +2166,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;
void* pPkVal = NULL;
int32_t pkLen = 0;
void* pPkVal = NULL;
int32_t pkLen = 0;
SColumnInfoData* pPkColDataInfo = NULL;
if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
@ -2453,8 +2472,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa
pBlock->info.id.groupId = pKey->groupId;
void* tbname = NULL;
if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId,
&tbname, false) < 0) {
if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,
false) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -2934,7 +2953,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) {
int32_t size = 0;
void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY));
@ -2967,7 +2986,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) {
int32_t size = 0;
void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY));
@ -3017,6 +3036,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
int32_t numOfCols = 0;
int32_t code = TSDB_CODE_OUT_OF_MEMORY;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -3111,8 +3131,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
}
return pOperator;
@ -3123,6 +3147,7 @@ _error:
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}
@ -3573,7 +3598,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (!allEqual) {
uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
@ -3680,9 +3705,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
SSessionKey key = {0};
SResultWindowInfo winfo = {0};
buf = decodeSSessionKey(buf, &key);
pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL,
pAggSup->stateKeySize, compareStateKey,
(void**)&winfo.pStatePos, &pAggSup->resultRowSize);
pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize,
compareStateKey, (void**)&winfo.pStatePos,
&pAggSup->resultRowSize);
buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
}
@ -3876,7 +3901,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) {
int32_t size = 0;
void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME,
strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size);
strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
qDebug("===stream=== reload state. get result count:%d", num);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
@ -3940,12 +3965,13 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey);
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
TSDB_CHECK_CODE(code, lino, _error);
}
pInfo->stateCol = extractColumnFromColumnNode(pColNode);
@ -3954,9 +3980,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
}
pInfo->twAggSup = (STimeWindowAggSupp){
@ -3983,9 +4007,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize,
type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);
pInfo->pDelIterator = NULL;
@ -3998,7 +4021,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) {
goto _error;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
if (pHandle) {
pInfo->isHistoryOp = pHandle->fillHistory;
@ -4025,17 +4049,20 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
return pOperator;
_error:
destroyStreamStateOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}
@ -4202,11 +4229,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
.precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
};
pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,
.calTrigger = pIntervalPhyNode->window.triggerType,
.maxTs = INT64_MIN,
.minTs = INT64_MAX,
.deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};
pInfo->twAggSup =
(STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,
.calTrigger = pIntervalPhyNode->window.triggerType,
.maxTs = INT64_MIN,
.minTs = INT64_MAX,
.deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};
ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
@ -4334,7 +4362,7 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
.ts = nextWin.skey,
.groupId = groupId,
};
void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));
void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));
int32_t index = -1;
SArray* chArray = NULL;
int32_t chId = 0;
@ -4406,8 +4434,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) {
int32_t size = taosArrayGetSize(wins);
for (int32_t i = 0; i < size; i++) {
SWinKey* winKey = taosArrayGet(wins, i);
void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));
SWinKey* winKey = taosArrayGet(wins, i);
void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));
if (!chIds) {
addPullWindow(pMidPullMap, winKey, numOfChild);
qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild);
@ -4610,5 +4638,6 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
void setStreamOperatorCompleted(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator);
qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo),
getStreamOpName(pOperator->operatorType), pOperator->status);
}

4
source/libs/executor/src/timewindowoperator.c
View File

@ -1136,9 +1136,9 @@ void destroyIntervalOperatorInfo(void* param) {
tdListFree(pInfo->binfo.resultRowInfo.openWindow);
pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
taosArrayDestroy(pInfo->pInterpCols);
pInfo->pInterpCols = NULL;
taosArrayDestroyEx(pInfo->pPrevValues, freeItem);
pInfo->pPrevValues = NULL;
cleanupGroupResInfo(&pInfo->groupResInfo);

3
source/libs/function/src/builtinsimpl.c
View File

@ -1441,6 +1441,9 @@ _stddev_over:
}
static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) {
if (IS_NULL_TYPE(pInput->type)) {
return;
}
pOutput->type = pInput->type;
if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) {
pOutput->quadraticISum += pInput->quadraticISum;

27
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -1061,6 +1061,13 @@ void nodesDestroyNode(SNode* pNode) {
pStmt->freeStbRowsCxtFunc(pStmt->pStbRowsCxt);
}
taosMemoryFreeClear(pStmt->pStbRowsCxt);
taosMemoryFreeClear(pStmt->pCreateTbInfo);
if (pStmt->destroyParseFileCxt) {
pStmt->destroyParseFileCxt(&pStmt->pParFileCxt);
}
taosCloseFile(&pStmt->fp);
break;
}
@ -1093,15 +1100,6 @@ void nodesDestroyNode(SNode* pNode) {
}
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE: {
SCreateSubTableFromFileClause* pStmt = (SCreateSubTableFromFileClause*)pNode;
if (pStmt->aCreateTbData) {
taosArrayDestroy(pStmt->aCreateTbData);
}
if (pStmt->aTagIndexs) {
taosArrayDestroy(pStmt->aTagIndexs);
}
if (pStmt->fp) {
taosCloseFile(&pStmt->fp);
}
nodesDestroyList(pStmt->pSpecificTags);
break;
}
@ -2205,6 +2203,17 @@ bool nodesIsRegularOp(const SOperatorNode* pOp) {
return false;
}
bool nodesIsMatchRegularOp(const SOperatorNode* pOp) {
switch (pOp->opType) {
case OP_TYPE_MATCH:
case OP_TYPE_NMATCH:
return true;
default:
break;
}
return false;
}
bool nodesIsBitwiseOp(const SOperatorNode* pOp) {
switch (pOp->opType) {
case OP_TYPE_BIT_AND:

1
source/libs/parser/inc/parInt.h
View File

@ -29,6 +29,7 @@ extern "C" {
#define QUERY_SMA_OPTIMIZE_ENABLE 1
int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData);
int32_t continueCreateTbFromFile(SParseContext* pCxt, SQuery** pQuery);
int32_t parse(SParseContext* pParseCxt, SQuery** pQuery);
int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache);
int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache);

6
source/libs/parser/inc/sql.y
View File

@ -353,6 +353,8 @@ end_opt(A) ::= END WITH TIMESTAMP NK_STRING(B).
cmd ::= CREATE TABLE not_exists_opt(A) full_table_name(B)
NK_LP column_def_list(C) NK_RP tags_def_opt(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); }
cmd ::= CREATE TABLE multi_create_clause(A). { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, A); }
cmd ::= CREATE TABLE not_exists_opt(B) USING full_table_name(C)
NK_LP tag_list_opt(D) NK_RP FILE NK_STRING(E). { pCxt->pRootNode = createCreateSubTableFromFileClause(pCxt, B, C, D, &E); }
cmd ::= CREATE STABLE not_exists_opt(A) full_table_name(B)
NK_LP column_def_list(C) NK_RP tags_def(D) table_options(E). { pCxt->pRootNode = createCreateTableStmt(pCxt, A, B, C, D, E); }
cmd ::= DROP TABLE multi_drop_clause(A). { pCxt->pRootNode = createDropTableStmt(pCxt, A); }
@ -385,15 +387,11 @@ alter_table_clause(A) ::=
%destructor multi_create_clause { nodesDestroyList($$); }
multi_create_clause(A) ::= create_subtable_clause(B). { A = createNodeList(pCxt, B); }
multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). { A = addNodeToList(pCxt, B, C); }
multi_create_clause(A) ::= create_from_file_clause(B). { A = createNodeList(pCxt, B); }
create_subtable_clause(A) ::=
not_exists_opt(B) full_table_name(C) USING full_table_name(D)
specific_cols_opt(E) TAGS NK_LP tags_literal_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); }
create_from_file_clause(A) ::= not_exists_opt(B) USING full_table_name(C)
NK_LP tag_list_opt(D) NK_RP FILE NK_STRING(E). { A = createCreateSubTableFromFileClause(pCxt, B, C, D, &E); }
%type multi_drop_clause { SNodeList* }
%destructor multi_drop_clause { nodesDestroyList($$); }
multi_drop_clause(A) ::= drop_table_clause(B). { A = createNodeList(pCxt, B); }

20
source/libs/parser/src/parAstParser.c
View File

@ -310,6 +310,24 @@ static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCre
return code;
}
static int32_t collectMetaKeyFromCreateSubTableFromFile(SCollectMetaKeyCxt* pCxt,
SCreateSubTableFromFileClause* pClause) {
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code =
reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->useDbName, NULL,
AUTH_TYPE_WRITE, pCxt->pMetaCache);
}
return code;
}
static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
@ -866,6 +884,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt);
case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt);
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE:
return collectMetaKeyFromCreateSubTableFromFile(pCxt, (SCreateSubTableFromFileClause*)pStmt);
case QUERY_NODE_DROP_TABLE_STMT:
return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt);
case QUERY_NODE_DROP_SUPER_TABLE_STMT:

101
source/libs/parser/src/parInsertSml.c
View File

@ -112,7 +112,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
for (int i = 0; i < tags->numOfBound; ++i) {
SSchema* pTagSchema = &pSchema[tags->pColIndex[i]];
SSmlKv* kv = taosArrayGet(cols, i);
if (kv == NULL){
code = TSDB_CODE_SML_INVALID_DATA;
uError("SML smlBuildTagRow error kv is null");
goto end;
}
if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 ||
kv->type != pTagSchema->type) {
code = TSDB_CODE_SML_INVALID_DATA;
@ -120,7 +124,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
goto end;
}
taosArrayPush(*tagName, pTagSchema->name);
if (taosArrayPush(*tagName, pTagSchema->name) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML smlBuildTagRow error push tag name");
goto end;
}
STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
// strcpy(val.colName, pTagSchema->name);
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY ||
@ -141,7 +149,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
goto end;
}
char buf[512] = {0};
snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
(void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
taosMemoryFree(p);
code = buildSyntaxErrMsg(msg, buf, kv->value);
goto end;
@ -149,9 +157,13 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
val.pData = p;
val.nData = output;
} else {
memcpy(&val.i64, &(kv->value), kv->length);
(void)memcpy(&val.i64, &(kv->value), kv->length);
}
if (taosArrayPush(pTagArray, &val) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML smlBuildTagRow error push tag array");
goto end;
}
taosArrayPush(pTagArray, &val);
}
code = tTagNew(pTagArray, 1, false, ppTag);
@ -166,20 +178,19 @@ end:
return code;
}
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) {
STableDataCxt* pTableCxt = NULL;
int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) {
SVCreateTbReq* pCreateTbReq = NULL;
int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false);
sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false);
if (ret != TSDB_CODE_SUCCESS) {
return NULL;
return ret;
}
ret = initTableColSubmitData(pTableCxt);
ret = initTableColSubmitData(*cxt);
if (ret != TSDB_CODE_SUCCESS) {
return NULL;
return ret;
}
return pTableCxt;
return TSDB_CODE_SUCCESS;
}
void clearColValArraySml(SArray* pCols) {
@ -197,6 +208,9 @@ void clearColValArraySml(SArray* pCols) {
int32_t smlBuildRow(STableDataCxt* pTableCxt) {
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
if (pRow == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS != ret) {
return ret;
@ -211,18 +225,22 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
int ret = TSDB_CODE_SUCCESS;
SSchema* pColSchema = schema + index;
SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
if (pVal == NULL) {
return TSDB_CODE_SUCCESS;
}
SSmlKv* kv = (SSmlKv*)data;
if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 ||
kv->type != pColSchema->type) {
ret = TSDB_CODE_SML_INVALID_DATA;
char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1);
if (tmp) {
memcpy(tmp, kv->key, kv->keyLen);
(void)memcpy(tmp, kv->key, kv->keyLen);
uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name,
pColSchema->name, tDataTypes[pColSchema->type].name);
taosMemoryFree(tmp);
} else {
uError("SML smlBuildCol out of memory");
ret = TSDB_CODE_OUT_OF_MEMORY;
}
goto end;
}
@ -256,9 +274,9 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
(void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
} else {
memcpy(&pVal->value.val, &(kv->value), kv->length);
(void)memcpy(&pVal->value.val, &(kv->value), kv->length);
}
pVal->flag = CV_FLAG_VALUE;
@ -276,8 +294,13 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
SVCreateTbReq* pCreateTblReq = NULL;
SArray* tagName = NULL;
insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags);
int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true);
int ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags);
if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "init bound cols error");
goto end;
}
ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true);
if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "bound tags error");
goto end;
@ -299,7 +322,11 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
ttl);
pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1);
memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);
if (pCreateTblReq->ctb.stbName == NULL){
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
(void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);
if (dataFormat) {
STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj,
@ -345,11 +372,18 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
for (int32_t r = 0; r < rowNum; ++r) {
void* rowData = taosArrayGetP(cols, r);
if (rowData == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
goto end;
}
// 1. set the parsed value from sql string
for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) {
SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]];
SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]);
if (pVal == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
goto end;
}
void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
if (p == NULL) {
continue;
@ -388,14 +422,22 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
if (NULL == pVal->value.pData) {
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
(void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
} else {
memcpy(&pVal->value.val, &(kv->value), kv->length);
(void)memcpy(&pVal->value.val, &(kv->value), kv->length);
}
pVal->flag = CV_FLAG_VALUE;
}
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
if (NULL == pRow) {
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS != ret) {
buildInvalidOperationMsg(&pBuf, "tRowBuild error");
@ -415,11 +457,12 @@ end:
return ret;
}
SQuery* smlInitHandle() {
int32_t smlInitHandle(SQuery** query) {
*query = NULL;
SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == pQuery) {
uError("create pQuery error");
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->haveResultSet = false;
@ -428,14 +471,22 @@ SQuery* smlInitHandle() {
if (NULL == stmt) {
uError("create SVnodeModifyOpStmt error");
qDestroyQuery(pQuery);
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (stmt->pTableBlockHashObj == NULL){
uError("create pTableBlockHashObj error");
qDestroyQuery(pQuery);
nodesDestroyNode((SNode*)stmt);
return TSDB_CODE_OUT_OF_MEMORY;
}
stmt->freeHashFunc = insDestroyTableDataCxtHashMap;
stmt->freeArrayFunc = insDestroyVgroupDataCxtList;
pQuery->pRoot = (SNode*)stmt;
return pQuery;
*query = pQuery;
return TSDB_CODE_SUCCESS;
}
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) {

445
source/libs/parser/src/parTranslater.c
View File

@ -11889,7 +11889,8 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN
return TSDB_CODE_SUCCESS;
}
static int32_t createParOperatorNode(EOperatorType opType, const char* pLeftCol, const char* pRightCol, SNode** ppResOp) {
static int32_t createParOperatorNode(EOperatorType opType, const char* pLeftCol, const char* pRightCol,
SNode** ppResOp) {
SOperatorNode* pOper = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR);
CHECK_POINTER_OUT_OF_MEM(pOper);
@ -12694,7 +12695,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla
}
static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFileClause* pStmt,
STableMeta* pSuperTableMeta) {
STableMeta* pSuperTableMeta, SArray* aTagIndexs) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t numOfTags = getNumOfTags(pSuperTableMeta);
@ -12705,7 +12706,7 @@ static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFil
return TSDB_CODE_OUT_OF_MEMORY;
}
bool tbnameFound = false;
bool tbnameFound = false;
SNode* pTagNode;
FOREACH(pTagNode, pStmt->pSpecificTags) {
@ -12757,14 +12758,14 @@ static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFil
goto _OUT;
}
if (NULL == taosArrayPush(pStmt->aTagIndexs, &idx)) {
if (NULL == taosArrayPush(aTagIndexs, &idx)) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OUT;
}
}
if (TSDB_CODE_SUCCESS == code && !tbnameFound) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_ERROR);
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_ERROR);
}
_OUT:
@ -12772,21 +12773,19 @@ _OUT:
return code;
}
typedef struct {
// refer
STableMeta* pSuperTableMeta;
SArray* pTagIndexs;
TdFilePtr fp;
typedef struct SParseFileContext {
SHashObj* pTbNameHash;
SArray* aTagNames;
bool tagNameFilled;
STableMeta* pStbMeta;
SArray* aTagIndexs;
// containers
SHashObj* pTbNameHash;
SArray* aTagNames;
bool tagNameFilled;
SArray* aTagVals;
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW];
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW];
SArray* aCreateTbData;
// per line
const char* pSql;
SArray* aTagVals;
STag* pTag;
SName ctbName;
SVgroupInfo vg;
@ -12810,48 +12809,48 @@ static int32_t fillVgroupInfo(SParseContext* pParseCxt, const SName* pName, SVgr
return code;
}
static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCtx) {
static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt) {
int32_t code = TSDB_CODE_SUCCESS;
int sz = taosArrayGetSize(pParFileCtx->pTagIndexs);
int32_t numOfTags = getNumOfTags(pParFileCtx->pSuperTableMeta);
uint8_t precision = getTableInfo(pParFileCtx->pSuperTableMeta).precision;
SSchema* pSchemas = getTableTagSchema(pParFileCtx->pSuperTableMeta);
int sz = taosArrayGetSize(pParFileCxt->aTagIndexs);
int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta);
uint8_t precision = getTableInfo(pParFileCxt->pStbMeta).precision;
SSchema* pSchemas = getTableTagSchema(pParFileCxt->pStbMeta);
for (int i = 0; i < sz; i++) {
const char* pSql = pParFileCtx->pSql;
const char* pSql = pParFileCxt->pSql;
int32_t pos = 0;
SToken token = tStrGetToken(pSql, &pos, true, NULL);
pParFileCtx->pSql += pos;
pParFileCxt->pSql += pos;
if (TK_NK_RP == token.type) {
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
break;
}
int16_t index = *(int16_t*)taosArrayGet(pParFileCtx->pTagIndexs, i);
int16_t index = *(int16_t*)taosArrayGet(pParFileCxt->aTagIndexs, i);
if (index < numOfTags) {
// parse tag
const SSchema* pTagSchema = &pSchemas[index];
code = checkAndTrimValue(&token, pParFileCtx->tmpTokenBuf, pMsgBuf, pTagSchema->type);
code = checkAndTrimValue(&token, pParFileCxt->tmpTokenBuf, pMsgBuf, pTagSchema->type);
if (TSDB_CODE_SUCCESS == code && TK_NK_VARIABLE == token.type) {
code = buildInvalidOperationMsg(pMsgBuf, "not expected row value");
}
if (TSDB_CODE_SUCCESS == code) {
SArray* aTagNames = pParFileCtx->tagNameFilled ? NULL : pParFileCtx->aTagNames;
code = parseTagValue(pMsgBuf, &pParFileCtx->pSql, precision, (SSchema*)pTagSchema, &token,
aTagNames, pParFileCtx->aTagVals, &pParFileCtx->pTag);
SArray* aTagNames = pParFileCxt->tagNameFilled ? NULL : pParFileCxt->aTagNames;
code = parseTagValue(pMsgBuf, &pParFileCxt->pSql, precision, (SSchema*)pTagSchema, &token, aTagNames,
pParFileCxt->aTagVals, &pParFileCxt->pTag);
}
} else {
// parse tbname
code = checkAndTrimValue(&token, pParFileCtx->tmpTokenBuf, pMsgBuf, TSDB_DATA_TYPE_BINARY);
code = checkAndTrimValue(&token, pParFileCxt->tmpTokenBuf, pMsgBuf, TSDB_DATA_TYPE_BINARY);
if (TK_NK_VARIABLE == token.type) {
code = buildInvalidOperationMsg(pMsgBuf, "not expected tbname");
}
if (TSDB_CODE_SUCCESS == code) {
bool bFoundTbName = false;
code = parseTbnameToken(pMsgBuf, pParFileCtx->ctbName.tname, &token, &bFoundTbName);
code = parseTbnameToken(pMsgBuf, pParFileCxt->ctbName.tname, &token, &bFoundTbName);
}
}
@ -12859,8 +12858,8 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCtx)
}
if (TSDB_CODE_SUCCESS == code) {
pParFileCtx->tagNameFilled = true;
code = tTagNew(pParFileCtx->aTagVals, 1, false, &pParFileCtx->pTag);
pParFileCxt->tagNameFilled = true;
code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag);
}
return code;
@ -12873,171 +12872,262 @@ typedef struct {
SVgroupInfo vg;
} SCreateTableData;
static void clearTagValArrayFp(void *data) {
static void clearTagValArrayFp(void* data) {
STagVal* p = (STagVal*)data;
if (IS_VAR_DATA_TYPE(p->type)) {
taosMemoryFreeClear(p->pData);
}
}
static void clearCreateTbArrayFp(void *data) {
static void clearCreateTbArrayFp(void* data) {
SCreateTableData* p = (SCreateTableData*)data;
taosMemoryFreeClear(p->pTag);
}
static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFileContext* pParseFileCtx,
SArray* aCreateTbData) {
static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFileContext* pParFileCxt, TdFilePtr fp,
int32_t maxLineCount) {
int32_t code = TSDB_CODE_SUCCESS;
char* pLine = NULL;
int64_t readLen = 0;
while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pParseFileCtx->fp, &pLine)) != -1) {
int32_t lineCount = 0;
while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(fp, &pLine)) != -1) {
if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) {
pLine[--readLen] = '\0';
}
if (readLen == 0) continue;
if (pLine[0] == '#') continue;
if (pLine[0] == '#') continue; // ignore comment line begins with '#'
strtolower(pLine, pLine);
pParseFileCtx->pSql = pLine;
pParFileCxt->pSql = pLine;
code = parseOneStbRow(pMsgBuf, pParseFileCtx);
code = parseOneStbRow(pMsgBuf, pParFileCxt);
if (TSDB_CODE_SUCCESS == code) {
if (taosHashGet(pParseFileCtx->pTbNameHash, pParseFileCtx->ctbName.tname,
strlen(pParseFileCtx->ctbName.tname) + 1) != NULL) {
taosMemoryFreeClear(pParseFileCtx->pTag);
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_DUPLICATED, pParseFileCtx->ctbName.tname);
if (taosHashGet(pParFileCxt->pTbNameHash, pParFileCxt->ctbName.tname, strlen(pParFileCxt->ctbName.tname) + 1) !=
NULL) {
taosMemoryFreeClear(pParFileCxt->pTag);
code = generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_TBNAME_DUPLICATED, pParFileCxt->ctbName.tname);
break;
}
code = taosHashPut(pParseFileCtx->pTbNameHash, pParseFileCtx->ctbName.tname,
strlen(pParseFileCtx->ctbName.tname) + 1, NULL, 0);
code = taosHashPut(pParFileCxt->pTbNameHash, pParFileCxt->ctbName.tname, strlen(pParFileCxt->ctbName.tname) + 1,
NULL, 0);
}
if (TSDB_CODE_SUCCESS == code) {
code = fillVgroupInfo(pParseCxt, &pParseFileCtx->ctbName, &pParseFileCtx->vg);
code = fillVgroupInfo(pParseCxt, &pParFileCxt->ctbName, &pParFileCxt->vg);
}
if (TSDB_CODE_SUCCESS == code) {
SCreateTableData data = {.ctbName = pParseFileCtx->ctbName,
.aTagNames = pParseFileCtx->aTagNames,
.pTag = pParseFileCtx->pTag,
.vg = pParseFileCtx->vg};
SCreateTableData data = {.ctbName = pParFileCxt->ctbName,
.aTagNames = pParFileCxt->aTagNames,
.pTag = pParFileCxt->pTag,
.vg = pParFileCxt->vg};
taosArrayPush(aCreateTbData, &data);
taosArrayPush(pParFileCxt->aCreateTbData, &data);
} else {
taosMemoryFreeClear(pParseFileCtx->pTag);
taosMemoryFreeClear(pParFileCxt->pTag);
}
pParseFileCtx->pTag = NULL;
taosArrayClearEx(pParseFileCtx->aTagVals, clearTagValArrayFp);
pParFileCxt->pTag = NULL;
taosArrayClearEx(pParFileCxt->aTagVals, clearTagValArrayFp);
lineCount++;
if (lineCount == maxLineCount) break;
}
if (TSDB_CODE_SUCCESS != code) {
taosArrayClearEx(aCreateTbData, clearCreateTbArrayFp);
taosArrayClearEx(pParFileCxt->aCreateTbData, clearCreateTbArrayFp);
}
taosMemoryFree(pLine);
return code;
}
static int32_t prepareReadFromFile(SCreateSubTableFromFileClause* pStmt) {
static void destructParseFileContext(SParseFileContext** ppParFileCxt) {
if (NULL == ppParFileCxt || NULL == *ppParFileCxt) {
return;
}
SParseFileContext* pParFileCxt = *ppParFileCxt;
taosHashCleanup(pParFileCxt->pTbNameHash);
taosArrayDestroy(pParFileCxt->aTagNames);
taosMemoryFreeClear(pParFileCxt->pStbMeta);
taosArrayDestroy(pParFileCxt->aTagIndexs);
taosArrayDestroy(pParFileCxt->aCreateTbData);
taosArrayDestroy(pParFileCxt->aTagVals);
taosMemoryFree(pParFileCxt);
*ppParFileCxt = NULL;
return;
}
static int32_t constructParseFileContext(SCreateSubTableFromFileClause* pStmt, STableMeta* pSuperTableMeta,
int32_t acctId, SParseFileContext** ppParFileCxt) {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pStmt->fp) {
pStmt->fp = taosOpenFile(pStmt->filePath, TD_FILE_READ | TD_FILE_STREAM);
if (NULL == pStmt->fp) {
SParseFileContext* pParFileCxt = taosMemoryCalloc(1, sizeof(SParseFileContext));
pParFileCxt->pStbMeta = pSuperTableMeta;
pParFileCxt->tagNameFilled = false;
pParFileCxt->pTag = NULL;
pParFileCxt->ctbName.type = TSDB_TABLE_NAME_T;
pParFileCxt->ctbName.acctId = acctId;
strcpy(pParFileCxt->ctbName.dbname, pStmt->useDbName);
if (NULL == pParFileCxt->pTbNameHash) {
pParFileCxt->pTbNameHash =
taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (!pParFileCxt->pTbNameHash) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
if (NULL == pParFileCxt->aTagNames) {
pParFileCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN);
if (NULL == pParFileCxt->aTagNames) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
if (NULL == pParFileCxt->aCreateTbData) {
pParFileCxt->aCreateTbData = taosArrayInit(16, sizeof(SCreateTableData));
if (NULL == pParFileCxt->aCreateTbData) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
if (NULL == pParFileCxt->aTagIndexs) {
pParFileCxt->aTagIndexs = taosArrayInit(pStmt->pSpecificTags->length, sizeof(int16_t));
if (!pParFileCxt->aTagIndexs) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
if (NULL == pParFileCxt->aTagVals) {
pParFileCxt->aTagVals = taosArrayInit(8, sizeof(STagVal));
if (!pParFileCxt->aTagVals) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
}
*ppParFileCxt = pParFileCxt;
return code;
_ERR:
destructParseFileContext(&pParFileCxt);
return code;
}
typedef struct SCreateTbInfo {
bool ignoreExists;
char useDbName[TSDB_DB_NAME_LEN];
char useTableName[TSDB_TABLE_NAME_LEN];
} SCreateTbInfo;
static int32_t prepareReadCsvFile(STranslateContext* pCxt, SCreateSubTableFromFileClause* pCreateStmt,
SVnodeModifyOpStmt* pModifyStmt) {
int32_t code = 0;
TdFilePtr fp = NULL;
SCreateTbInfo* pCreateInfo = NULL;
SParseFileContext* pParFileCxt = NULL;
if (NULL == pModifyStmt->fp) {
fp = taosOpenFile(pCreateStmt->filePath, TD_FILE_READ | TD_FILE_STREAM);
if (NULL == fp) {
code = TAOS_SYSTEM_ERROR(errno);
goto _ERR;
}
}
if (NULL == pStmt->aCreateTbData) {
pStmt->aCreateTbData = taosArrayInit(16, sizeof(SCreateTableData));
if (NULL == pStmt->aCreateTbData) {
{
pCreateInfo = taosMemoryCalloc(1, sizeof(SCreateTbInfo));
if (NULL == pCreateInfo) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
pCreateInfo->ignoreExists = pCreateStmt->ignoreExists;
strncpy(pCreateInfo->useDbName, pCreateStmt->useDbName, TSDB_DB_NAME_LEN);
strncpy(pCreateInfo->useTableName, pCreateStmt->useTableName, TSDB_TABLE_NAME_LEN);
}
if (NULL == pStmt->aTagIndexs) {
pStmt->aTagIndexs = taosArrayInit(pStmt->pSpecificTags->length, sizeof(int16_t));
if (!pStmt->aTagIndexs) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _ERR;
}
{
STableMeta* pSuperTableMeta = NULL;
code = getTableMeta(pCxt, pCreateStmt->useDbName, pCreateStmt->useTableName, &pSuperTableMeta);
if (TSDB_CODE_SUCCESS != code) goto _ERR;
code = constructParseFileContext(pCreateStmt, pSuperTableMeta, pCxt->pParseCxt->acctId, &pParFileCxt);
if (TSDB_CODE_SUCCESS != code) goto _ERR;
code = buildTagIndexForBindTags(&pCxt->msgBuf, pCreateStmt, pParFileCxt->pStbMeta, pParFileCxt->aTagIndexs);
if (TSDB_CODE_SUCCESS != code) goto _ERR;
}
pModifyStmt->fp = fp;
pModifyStmt->fileProcessing = false;
pModifyStmt->pCreateTbInfo = pCreateInfo;
pModifyStmt->pParFileCxt = pParFileCxt;
return code;
_ERR:
taosCloseFile(&pStmt->fp);
taosArrayDestroy(pStmt->aCreateTbData);
taosArrayDestroy(pStmt->aTagIndexs);
taosCloseFile(&fp);
taosMemoryFreeClear(pCreateInfo);
destructParseFileContext(&pParFileCxt);
return code;
}
static int32_t rewriteCreateSubTableFromFile(STranslateContext* pCxt, SCreateSubTableFromFileClause* pStmt,
SHashObj* pVgroupHashmap) {
static int32_t resetParseFileContext(SParseFileContext* pParFileCxt) {
taosArrayClear(pParFileCxt->aCreateTbData);
taosArrayClearEx(pParFileCxt->aTagVals, clearTagValArrayFp);
return TSDB_CODE_SUCCESS;
}
static int32_t createSubTableFromFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SVnodeModifyOpStmt* pModifyStmt) {
int32_t code = 0;
STableMeta* pSuperTableMeta = NULL;
if (TSDB_CODE_SUCCESS == code) {
code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta);
}
SCreateTbInfo* pCreateInfo = pModifyStmt->pCreateTbInfo;
SParseFileContext* pParFileCxt = pModifyStmt->pParFileCxt;
if (TSDB_CODE_SUCCESS == code) {
code = prepareReadFromFile(pStmt);
code = parseCsvFile(pMsgBuf, pParseCxt, pParFileCxt, pModifyStmt->fp, tsMaxInsertBatchRows);
}
STableMeta* pSuperTableMeta = pParFileCxt->pStbMeta;
if (TSDB_CODE_SUCCESS == code) {
code = buildTagIndexForBindTags(&pCxt->msgBuf, pStmt, pSuperTableMeta);
}
SParseFileContext parseFileCtx = {
.pSuperTableMeta = pSuperTableMeta, .fp = pStmt->fp, .pTagIndexs = pStmt->aTagIndexs};
parseFileCtx.pTbNameHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
parseFileCtx.aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN);
parseFileCtx.tagNameFilled = false;
parseFileCtx.aTagVals = taosArrayInit(8, sizeof(STagVal));
parseFileCtx.pTag = NULL;
parseFileCtx.ctbName.type = TSDB_TABLE_NAME_T;
parseFileCtx.ctbName.acctId = pCxt->pParseCxt->acctId;
strcpy(parseFileCtx.ctbName.dbname, pStmt->useDbName);
if (NULL == parseFileCtx.aTagNames || NULL == parseFileCtx.aTagVals || NULL == parseFileCtx.pTbNameHash) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OUT;
}
if (TSDB_CODE_SUCCESS == code) {
code = parseCsvFile(&pCxt->msgBuf, pCxt->pParseCxt, &parseFileCtx, pStmt->aCreateTbData);
}
if (TSDB_CODE_SUCCESS == code) {
int sz = taosArrayGetSize(pStmt->aCreateTbData);
int sz = taosArrayGetSize(pParFileCxt->aCreateTbData);
for (int i = 0; i < sz; i++) {
SCreateTableData* pData = taosArrayGet(pStmt->aCreateTbData, i);
SCreateTableData* pData = taosArrayGet(pParFileCxt->aCreateTbData, i);
code = collectUseTable(&pData->ctbName, pCxt->pTargetTables);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(pData->pTag);
}
// code = collectUseTable(&pData->ctbName, pCxt->pTargetTables);
// if (TSDB_CODE_SUCCESS != code) {
// taosMemoryFree(pData->pTag);
// }
code = addCreateTbReqIntoVgroup(pVgroupHashmap, pStmt->useDbName, pSuperTableMeta->uid, pStmt->useTableName,
pData->ctbName.tname, pData->aTagNames, pSuperTableMeta->tableInfo.numOfTags,
pData->pTag, TSDB_DEFAULT_TABLE_TTL, NULL, pStmt->ignoreExists, &pData->vg);
code = addCreateTbReqIntoVgroup(pModifyStmt->pVgroupsHashObj, pCreateInfo->useDbName, pSuperTableMeta->uid,
pCreateInfo->useTableName, pData->ctbName.tname, pData->aTagNames,
pSuperTableMeta->tableInfo.numOfTags, pData->pTag, TSDB_DEFAULT_TABLE_TTL, NULL,
pCreateInfo->ignoreExists, &pData->vg);
}
if (TSDB_CODE_SUCCESS == code) {
pModifyStmt->fileProcessing = (sz == tsMaxInsertBatchRows);
}
}
_OUT:
taosMemoryFreeClear(pSuperTableMeta);
taosHashCleanup(parseFileCtx.pTbNameHash);
taosArrayDestroy(parseFileCtx.aTagNames);
taosArrayDestroy(parseFileCtx.aTagVals);
(void)resetParseFileContext(pModifyStmt->pParFileCxt);
return code;
}
@ -13074,13 +13164,8 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode;
FOREACH(pNode, pStmt->pSubTables) {
if (pNode->type == QUERY_NODE_CREATE_SUBTABLE_CLAUSE) {
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
code = rewriteCreateSubTable(pCxt, pClause, pVgroupHashmap);
} else {
SCreateSubTableFromFileClause* pClause = (SCreateSubTableFromFileClause*)pNode;
code = rewriteCreateSubTableFromFile(pCxt, pClause, pVgroupHashmap);
}
SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode;
code = rewriteCreateSubTable(pCxt, pClause, pVgroupHashmap);
if (TSDB_CODE_SUCCESS != code) {
taosHashCleanup(pVgroupHashmap);
return code;
@ -13096,6 +13181,75 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
return rewriteToVnodeModifyOpStmt(pQuery, pBufArray);
}
static int32_t rewriteCreateTableFromFile(STranslateContext* pCxt, SQuery* pQuery) {
SVnodeModifyOpStmt* pModifyStmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT);
if (pModifyStmt == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pModifyStmt->sqlNodeType = nodeType(pQuery->pRoot);
pModifyStmt->fileProcessing = false;
pModifyStmt->destroyParseFileCxt = destructParseFileContext;
pModifyStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (NULL == pModifyStmt->pVgroupsHashObj) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosHashSetFreeFp(pModifyStmt->pVgroupsHashObj, destroyCreateTbReqBatch);
SCreateSubTableFromFileClause* pCreateStmt = (SCreateSubTableFromFileClause*)pQuery->pRoot;
int32_t code = prepareReadCsvFile(pCxt, pCreateStmt, pModifyStmt);
if (TSDB_CODE_SUCCESS != code) {
taosHashCleanup(pModifyStmt->pVgroupsHashObj);
return code;
}
code = createSubTableFromFile(&pCxt->msgBuf, pCxt->pParseCxt, pModifyStmt);
if (TSDB_CODE_SUCCESS != code) {
taosHashCleanup(pModifyStmt->pVgroupsHashObj);
return code;
}
SArray* pBufArray = serializeVgroupsCreateTableBatch(pModifyStmt->pVgroupsHashObj);
taosHashClear(pModifyStmt->pVgroupsHashObj);
if (NULL == pBufArray) {
taosHashCleanup(pModifyStmt->pVgroupsHashObj);
return TSDB_CODE_OUT_OF_MEMORY;
}
pModifyStmt->pDataBlocks = pBufArray;
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = (SNode*)pModifyStmt;
return TSDB_CODE_SUCCESS;
}
int32_t continueCreateTbFromFile(SParseContext* pParseCxt, SQuery** pQuery) {
SVnodeModifyOpStmt* pModifyStmt = (SVnodeModifyOpStmt*)(*pQuery)->pRoot;
SMsgBuf tmpBuf = {0};
tmpBuf.buf = taosMemoryMalloc(1024);
int32_t code = createSubTableFromFile(&tmpBuf, pParseCxt, pModifyStmt);
if (TSDB_CODE_SUCCESS != code) goto _OUT;
SArray* pBufArray = serializeVgroupsCreateTableBatch(pModifyStmt->pVgroupsHashObj);
taosHashClear(pModifyStmt->pVgroupsHashObj);
if (NULL == pBufArray) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OUT;
}
pModifyStmt->pDataBlocks = pBufArray;
(*pQuery)->execStage = QUERY_EXEC_STAGE_SCHEDULE;
if (!pModifyStmt->fileProcessing) {
(*pQuery)->execMode = QUERY_EXEC_MODE_EMPTY_RESULT;
}
code = TSDB_CODE_SUCCESS;
_OUT:
taosMemoryFreeClear(tmpBuf.buf);
return code;
}
typedef struct SVgroupDropTableBatch {
SVDropTbBatchReq req;
SVgroupInfo info;
@ -13798,7 +13952,8 @@ static int32_t createParWhenThenNode(SNode* pWhen, SNode* pThen, SNode** ppResWh
return TSDB_CODE_SUCCESS;
}
static int32_t createParCaseWhenNode(SNode* pCase, SNodeList* pWhenThenList, SNode* pElse, const char* pAias, SNode** ppResCaseWhen) {
static int32_t createParCaseWhenNode(SNode* pCase, SNodeList* pWhenThenList, SNode* pElse, const char* pAias,
SNode** ppResCaseWhen) {
SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)nodesMakeNode(QUERY_NODE_CASE_WHEN);
CHECK_POINTER_OUT_OF_MEM(pCaseWhen);
@ -13813,7 +13968,8 @@ static int32_t createParCaseWhenNode(SNode* pCase, SNodeList* pWhenThenList, SNo
return TSDB_CODE_SUCCESS;
}
static int32_t createParFunctionNode(const char* pFunName, const char* pAias, SNodeList* pParameterList, SNode** ppResFunc) {
static int32_t createParFunctionNode(const char* pFunName, const char* pAias, SNodeList* pParameterList,
SNode** ppResFunc) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
CHECK_POINTER_OUT_OF_MEM(pFunc);
strcpy(pFunc->functionName, pFunName);
@ -13854,13 +14010,13 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
}
}
SValueNode* pValNode = nodesMakeValueNodeFromString("leader");
SValueNode* pValNode = nodesMakeValueNodeFromString("leader");
CHECK_POINTER_OUT_OF_MEM(pValNode);
SNode* pCond1 = NULL;
SNode* pCond2 = NULL;
SNode* pCond3 = NULL;
SNode* pCond4 = NULL;
SNode* pCond1 = NULL;
SNode* pCond2 = NULL;
SNode* pCond3 = NULL;
SNode* pCond4 = NULL;
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "v1_status", (SNode*)pValNode, &pCond1));
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "v2_status", (SNode*)pValNode, &pCond2));
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "v3_status", (SNode*)pValNode, &pCond3));
@ -13885,16 +14041,16 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
SNode* pElse = nodesMakeValueNodeFromInt32(0);
CHECK_POINTER_OUT_OF_MEM(pElse);
// case when (v1_status = "leader" or v2_status = "lead er" or v3_status = "leader" or v4_status = "leader") then 1 else 0 end
// case when (v1_status = "leader" or v2_status = "lead er" or v3_status = "leader" or v4_status = "leader") then 1
// else 0 end
SNode* pCaseWhen = NULL;
CHECK_RES_OUT_OF_MEM(createParCaseWhenNode(NULL, pWhenThenlist, pElse, NULL, &pCaseWhen));
SNodeList* pParaList = NULL;
CHECK_RES_OUT_OF_MEM(createParListNode(pCaseWhen, &pParaList));
// sum( case when ... end) as leader_col
SNode* pSumFun = NULL;
SNode* pSumFun = NULL;
const char* pSumColAlias = "leader_col";
CHECK_RES_OUT_OF_MEM(createParFunctionNode("sum", pSumColAlias, pParaList, &pSumFun));
@ -13904,7 +14060,7 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
CHECK_RES_OUT_OF_MEM(createParListNode(pPara1, &pParaList));
// count(1) as count_col
SNode* pCountFun = NULL;
SNode* pCountFun = NULL;
const char* pCountColAlias = "count_col";
CHECK_RES_OUT_OF_MEM(createParFunctionNode("count", pCountColAlias, pParaList, &pCountFun));
@ -13914,11 +14070,13 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
SSelectStmt* pSubSelect = NULL;
// select sum( case when .... end) as leader_col, count(*) as count_col from information_schema.ins_vgroups
CHECK_RES_OUT_OF_MEM(createSimpleSelectStmtFromProjList(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VGROUPS, pProjList, &pSubSelect));
CHECK_RES_OUT_OF_MEM(
createSimpleSelectStmtFromProjList(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VGROUPS, pProjList, &pSubSelect));
if (pDbName && pDbName[0] != 0) {
// for show db.alive
// select sum( case when .... end) as leader_col, count(*) as count_col from information_schema.ins_vgroups where db_name = "..."
// select sum( case when .... end) as leader_col, count(*) as count_col from information_schema.ins_vgroups where
// db_name = "..."
SNode* pDbCond = NULL;
pValNode = nodesMakeValueNodeFromString(pDbName);
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_EQUAL, "db_name", (SNode*)pValNode, &pDbCond));
@ -13928,14 +14086,12 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
pSubSelect->pWhere = pDbCond;
}
pCond1 = NULL;
CHECK_RES_OUT_OF_MEM(createParOperatorNode(OP_TYPE_EQUAL, pSumColAlias, pCountColAlias, &pCond1));
pCond2 = NULL;
SNode* pTempVal = nodesMakeValueNodeFromInt32(0);
CHECK_RES_OUT_OF_MEM(createOperatorNode(OP_TYPE_GREATER_THAN, pSumColAlias, pTempVal, &pCond2));
//leader_col = count_col and leader_col > 0
// leader_col = count_col and leader_col > 0
pTemp1 = NULL;
CHECK_RES_OUT_OF_MEM(createLogicCondNode(pCond1, pCond2, &pTemp1, LOGIC_COND_TYPE_AND));
@ -13961,7 +14117,8 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
CHECK_RES_OUT_OF_MEM(createParWhenThenNode(pTemp2, pThen, &pWhenThen));
CHECK_RES_OUT_OF_MEM(nodesListStrictAppend(pWhenThenlist, pWhenThen));
// case when leader_col = count_col and count_col > 0 then 1 when leader_col < count_col and count_col > 0 then 2 else 0 end as status
// case when leader_col = count_col and count_col > 0 then 1 when leader_col < count_col and count_col > 0 then 2 else
// 0 end as status
pCaseWhen = NULL;
pElse = nodesMakeValueNodeFromInt32(0);
CHECK_POINTER_OUT_OF_MEM(pElse);
@ -13973,7 +14130,6 @@ static int32_t rewriteShowAliveStmt(STranslateContext* pCxt, SQuery* pQuery) {
SNode* pTempTblNode = NULL;
CHECK_RES_OUT_OF_MEM(createParTempTableNode(pSubSelect, &pTempTblNode));
SSelectStmt* pStmt = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT);
CHECK_POINTER_OUT_OF_MEM(pStmt);
pStmt->pProjectionList = pProjList;
@ -14047,6 +14203,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
code = rewriteCreateMultiTable(pCxt, pQuery);
break;
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE:
code = rewriteCreateTableFromFile(pCxt, pQuery);
break;
case QUERY_NODE_DROP_TABLE_STMT:
code = rewriteDropTable(pCxt, pQuery);
break;

4
source/libs/parser/src/parUtil.c
View File

@ -223,7 +223,9 @@ static char* getSyntaxErrFormat(int32_t errCode) {
return "Tag name:%s duplicated";
case TSDB_CODE_PAR_NOT_ALLOWED_DIFFERENT_BY_ROW_FUNC:
return "Some functions cannot appear in the select list at the same time";
default:
case TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR:
return "Syntax error in regular expression";
default:
return "Unknown error";
}
}

37
source/libs/parser/src/parser.c
View File

@ -48,6 +48,33 @@ bool qIsInsertValuesSql(const char* pStr, size_t length) {
return false;
}
bool qIsCreateTbFromFileSql(const char* pStr, size_t length) {
if (NULL == pStr) {
return false;
}
const char* pSql = pStr;
int32_t index = 0;
SToken t = tStrGetToken((char*)pStr, &index, false, NULL);
if (TK_CREATE != t.type) {
return false;
}
do {
pStr += index;
index = 0;
t = tStrGetToken((char*)pStr, &index, false, NULL);
if (TK_FILE == t.type) {
return true;
}
if (0 == t.type || 0 == t.n) {
break;
}
} while (pStr - pSql < length);
return false;
}
bool qParseDbName(const char* pStr, size_t length, char** pDbName) {
(void) length;
int32_t index = 0;
@ -239,11 +266,19 @@ static int32_t parseQuerySyntax(SParseContext* pCxt, SQuery** pQuery, struct SCa
return code;
}
static int32_t parseCreateTbFromFileSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) {
if (NULL == *pQuery) return parseQuerySyntax(pCxt, pQuery, pCatalogReq);
return continueCreateTbFromFile(pCxt, pQuery);
}
int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) {
int32_t code = nodesAcquireAllocator(pCxt->allocatorId);
if (TSDB_CODE_SUCCESS == code) {
if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) {
code = parseInsertSql(pCxt, pQuery, pCatalogReq, NULL);
} else if (qIsCreateTbFromFileSql(pCxt->pSql, pCxt->sqlLen)) {
code = parseCreateTbFromFileSyntax(pCxt, pQuery, pCatalogReq);
} else {
code = parseQuerySyntax(pCxt, pQuery, pCatalogReq);
}
@ -322,7 +357,7 @@ void destoryCatalogReq(SCatalogReq *pCatalogReq) {
taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq);
#ifdef TD_ENTERPRISE
taosArrayDestroyEx(pCatalogReq->pView, destoryTablesReq);
#endif
#endif
taosArrayDestroyEx(pCatalogReq->pTableTSMAs, destoryTablesReq);
taosArrayDestroyEx(pCatalogReq->pTSMAs, destoryTablesReq);
}

9963
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

9
source/libs/planner/src/planLogicCreater.c
View File

@ -539,7 +539,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
pJoin->pJLimit = nodesCloneNode(pJoinTable->pJLimit);
pJoin->addPrimEqCond = nodesCloneNode(pJoinTable->addPrimCond);
pJoin->node.pChildren = nodesMakeList();
pJoin->seqWinGroup = (JOIN_STYPE_WIN == pJoinTable->subType) && (pSelect->hasAggFuncs || pSelect->hasIndefiniteRowsFunc);
pJoin->seqWinGroup = (JOIN_STYPE_WIN == pJoinTable->subType) && (pSelect->hasAggFuncs || pSelect->hasIndefiniteRowsFunc);
if (NULL == pJoin->node.pChildren) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -619,7 +619,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
}
}
#else
#else
// set the output
if (TSDB_CODE_SUCCESS == code) {
SNodeList* pColList = NULL;
@ -762,7 +762,7 @@ static int32_t addWinJoinPrimKeyToAggFuncs(SSelectStmt* pSelect, SNodeList** pLi
pCol->hasIndex = (pColSchema != NULL && IS_IDX_ON(pColSchema));
pCol->node.resType.type = pColSchema->type;
pCol->node.resType.bytes = pColSchema->bytes;
pCol->node.resType.precision = pProbeTable->pMeta->tableInfo.precision;
pCol->node.resType.precision = pProbeTable->pMeta->tableInfo.precision;
SNode* pFunc = (SNode*)createGroupKeyAggFunc(pCol);
@ -841,7 +841,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0;
pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0;
pAgg->hasGroup = pAgg->pGroupKeys || pSelect->pPartitionByList;
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pAgg;
} else {
@ -1708,6 +1708,7 @@ static int32_t getMsgType(ENodeType sqlType) {
switch (sqlType) {
case QUERY_NODE_CREATE_TABLE_STMT:
case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
case QUERY_NODE_CREATE_SUBTABLE_FROM_FILE_CLAUSE:
return TDMT_VND_CREATE_TABLE;
case QUERY_NODE_DROP_TABLE_STMT:
return TDMT_VND_DROP_TABLE;

8
source/libs/qworker/inc/qwInt.h
View File

@ -218,8 +218,8 @@ typedef struct SQWorkerMgmt {
#define QW_IDS() sId, qId, tId, rId, eId
#define QW_FPARAMS() mgmt, QW_IDS()
#define QW_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n)
#define QW_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n)
#define QW_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n)
#define QW_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n)
#define QW_STAT_GET(_item) atomic_load_64(&(_item))
#define QW_GET_EVENT(ctx, event) atomic_load_8(&(ctx)->events[event])
@ -250,7 +250,7 @@ typedef struct SQWorkerMgmt {
} while (0)
#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code)
#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_UPDATE_RSP_CODE(ctx, code) (void)atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY)
#define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch)
@ -391,7 +391,7 @@ void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx);
int32_t qwKillTaskHandle(SQWTaskCtx *ctx, int32_t rspCode);
int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status, bool dynamicTask);
int32_t qwDropTask(QW_FPARAMS_DEF);
void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx);
int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx);
int32_t qwOpenRef(void);
void qwSetHbParam(int64_t refId, SQWHbParam **pParam);
int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type);

31
source/libs/qworker/src/qwDbg.c
View File

@ -173,8 +173,21 @@ int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int
if (pEpSet) {
contLen = tSerializeSEpSet(NULL, 0, pEpSet);
if (contLen < 0) {
qError("tSerializeSEpSet failed, code:%x", terrno);
return terrno;
}
rsp = rpcMallocCont(contLen);
tSerializeSEpSet(rsp, contLen, pEpSet);
if (NULL == rsp) {
qError("rpcMallocCont %d failed, code:%x", contLen, terrno);
return terrno;
}
contLen = tSerializeSEpSet(rsp, contLen, pEpSet);
if (contLen < 0) {
qError("tSerializeSEpSet second failed, code:%x", terrno);
return terrno;
}
}
SRpcMsg rpcRsp = {
@ -208,28 +221,28 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) {
SEpSet epSet = {0};
epSet.inUse = 1;
epSet.numOfEps = 3;
strcpy(epSet.eps[0].fqdn, "localhost");
TAOS_STRCPY(epSet.eps[0].fqdn, "localhost");
epSet.eps[0].port = 7100;
strcpy(epSet.eps[1].fqdn, "localhost");
TAOS_STRCPY(epSet.eps[1].fqdn, "localhost");
epSet.eps[1].port = 7200;
strcpy(epSet.eps[2].fqdn, "localhost");
TAOS_STRCPY(epSet.eps[2].fqdn, "localhost");
epSet.eps[2].port = 7300;
ctx->phase = QW_PHASE_POST_QUERY;
qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, &epSet);
(void)qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, &epSet); // ignore error
*rsped = true;
return;
}
if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) {
QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY);
qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL);
(void)qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); // ignore error
*rsped = true;
return;
}
if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 9)) {
qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL);
(void)qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); // ignore error
*rsped = true;
return;
}
@ -260,8 +273,8 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) {
if (++ignoreTime > 10 && 0 == taosRand() % 9) {
if (ctx->fetchMsgType == TDMT_SCH_FETCH) {
qwBuildAndSendErrorRsp(TDMT_SCH_LINK_BROKEN, &ctx->ctrlConnInfo, TSDB_CODE_RPC_BROKEN_LINK);
qwBuildAndSendErrorRsp(ctx->fetchMsgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST);
(void)qwBuildAndSendErrorRsp(TDMT_SCH_LINK_BROKEN, &ctx->ctrlConnInfo, TSDB_CODE_RPC_BROKEN_LINK); // ignore error
(void)qwBuildAndSendErrorRsp(ctx->fetchMsgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); // ignore error
*rsped = true;
taosSsleep(3);

83
source/libs/qworker/src/qwMsg.c
View File

@ -21,7 +21,7 @@ int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **r
}
if (NULL == *rsp) {
memset(pRsp, 0, sizeof(SRetrieveTableRsp));
TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp));
}
*rsp = pRsp;
@ -105,8 +105,19 @@ int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SArray *pExecList) {
SExplainRsp rsp = {.numOfPlans = taosArrayGetSize(pExecList), .subplanInfo = pInfo};
int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp);
if (contLen < 0) {
qError("tSerializeSExplainRsp failed, error: %x", terrno);
QW_RET(terrno);
}
void *pRsp = rpcMallocCont(contLen);
tSerializeSExplainRsp(pRsp, contLen, &rsp);
if (NULL == pRsp) {
QW_RET(terrno);
}
contLen = tSerializeSExplainRsp(pRsp, contLen, &rsp);
if (contLen < 0) {
qError("tSerializeSExplainRsp second failed, error: %x", terrno);
QW_RET(terrno);
}
SRpcMsg rpcRsp = {
.msgType = TDMT_SCH_EXPLAIN_RSP,
@ -123,8 +134,20 @@ int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SArray *pExecList) {
int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus);
if (contLen < 0) {
qError("tSerializeSSchedulerHbRsp failed, error: %x", terrno);
QW_RET(terrno);
}
void *pRsp = rpcMallocCont(contLen);
tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus);
if (NULL == pRsp) {
QW_RET(terrno);
}
contLen = tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus);
if (contLen < 0) {
qError("tSerializeSSchedulerHbRsp second failed, error: %x", terrno);
QW_RET(terrno);
}
SRpcMsg rpcRsp = {
.msgType = TDMT_SCH_QUERY_HEARTBEAT_RSP,
@ -143,7 +166,10 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve
int32_t code) {
if (NULL == pRsp) {
pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
memset(pRsp, 0, sizeof(SRetrieveTableRsp));
if (NULL == pRsp) {
QW_RET(terrno);
}
TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp));
dataLength = 0;
}
@ -164,6 +190,9 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve
#if 0
int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code) {
STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp));
if (NULL == pRsp) {
QW_RET(terrno);
}
pRsp->code = code;
SRpcMsg rpcRsp = {
@ -180,6 +209,9 @@ int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code) {
int32_t qwBuildAndSendDropRsp(SRpcHandleInfo *pConn, int32_t code) {
STaskDropRsp *pRsp = (STaskDropRsp *)rpcMallocCont(sizeof(STaskDropRsp));
if (NULL == pRsp) {
QW_RET(terrno);
}
pRsp->code = code;
SRpcMsg rpcRsp = {
@ -405,6 +437,7 @@ int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) {
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
int32_t code = 0;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
SSubQueryMsg msg = {0};
if (tDeserializeSSubQueryMsg(pMsg->pCont, pMsg->contLen, &msg) < 0) {
@ -419,8 +452,8 @@ int32_t qWorkerAbortPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) {
int32_t eId = msg.execId;
QW_SCH_TASK_DLOG("Abort prerocessQuery start, handle:%p", pMsg->info.handle);
qwAbortPrerocessQuery(QW_FPARAMS());
QW_SCH_TASK_DLOG("Abort prerocessQuery end, handle:%p", pMsg->info.handle);
code = qwAbortPrerocessQuery(QW_FPARAMS());
QW_SCH_TASK_DLOG("Abort prerocessQuery end, handle:%p, code:%x", pMsg->info.handle, code);
tFreeSSubQueryMsg(&msg);
@ -435,7 +468,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
int32_t code = 0;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE);
QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.queryProcessed, 1);
SSubQueryMsg msg = {0};
@ -477,7 +510,7 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
SQWTaskCtx *handles = NULL;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE);
QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.cqueryProcessed, 1);
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
@ -495,9 +528,9 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
QW_SCH_TASK_DLOG("processCQuery start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessCQuery(QW_FPARAMS(), &qwMsg));
code = qwProcessCQuery(QW_FPARAMS(), &qwMsg);
QW_SCH_TASK_DLOG("processCQuery end, node:%p", node);
QW_SCH_TASK_DLOG("processCQuery end, node:%p, code:0x%x", node, code);
return TSDB_CODE_SUCCESS;
}
@ -510,7 +543,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
SResFetchReq req = {0};
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE);
QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.fetchProcessed, 1);
if (tDeserializeSResFetchReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
@ -528,9 +561,9 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
QW_SCH_TASK_DLOG("processFetch start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessFetch(QW_FPARAMS(), &qwMsg));
int32_t code = qwProcessFetch(QW_FPARAMS(), &qwMsg);
QW_SCH_TASK_DLOG("processFetch end, node:%p", node);
QW_SCH_TASK_DLOG("processFetch end, node:%p, code:%x", node, code);
return TSDB_CODE_SUCCESS;
}
@ -538,7 +571,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int
int32_t qWorkerProcessRspMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) {
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
if (mgmt) {
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE);
QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.rspProcessed, 1);
}
@ -557,7 +590,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
int32_t code = 0;
STaskCancelReq *msg = pMsg->pCont;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE);
QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.cancelProcessed, 1);
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
@ -598,7 +631,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6
int32_t code = 0;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE);
QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.dropProcessed, 1);
STaskDropReq msg = {0};
@ -621,9 +654,9 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6
QW_SCH_TASK_DLOG("processDrop start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessDrop(QW_FPARAMS(), &qwMsg));
code = qwProcessDrop(QW_FPARAMS(), &qwMsg);
QW_SCH_TASK_DLOG("processDrop end, node:%p", node);
QW_SCH_TASK_DLOG("processDrop end, node:%p, code:%x", node, code);
return TSDB_CODE_SUCCESS;
}
@ -636,7 +669,7 @@ int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
int32_t code = 0;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE);
QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.notifyProcessed, 1);
STaskNotifyReq msg = {0};
@ -655,9 +688,9 @@ int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
QW_SCH_TASK_DLOG("processNotify start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessNotify(QW_FPARAMS(), &qwMsg));
code = qwProcessNotify(QW_FPARAMS(), &qwMsg);
QW_SCH_TASK_DLOG("processNotify end, node:%p", node);
QW_SCH_TASK_DLOG("processNotify end, node:%p, code:%x", node, code);
return TSDB_CODE_SUCCESS;
}
@ -672,7 +705,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_
SSchedulerHbReq req = {0};
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE);
QW_ERR_RET(qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE));
QW_STAT_INC(mgmt->stat.msgStat.hbProcessed, 1);
if (NULL == pMsg->pCont) {
@ -694,9 +727,9 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_
QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessHb(mgmt, &qwMsg, &req));
code = qwProcessHb(mgmt, &qwMsg, &req);
QW_SCH_DLOG("processHb end, node:%p", node);
QW_SCH_DLOG("processHb end, node:%p, code:%x", node, code);
return TSDB_CODE_SUCCESS;
}
@ -712,7 +745,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD
QW_STAT_INC(mgmt->stat.msgStat.deleteProcessed, 1);
tDeserializeSVDeleteReq(pMsg->pCont, pMsg->contLen, &req);
QW_ERR_RET(tDeserializeSVDeleteReq(pMsg->pCont, pMsg->contLen, &req));
uint64_t sId = req.sId;
uint64_t qId = req.queryId;

66
source/libs/qworker/src/qwUtil.c
View File

@ -323,34 +323,52 @@ static void freeExplainExecItem(void *param) {
int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
int32_t code = TSDB_CODE_SUCCESS;
qTaskInfo_t taskHandle = ctx->taskHandle;
ctx->explainRsped = true;
SArray *execInfoList = taosArrayInit(4, sizeof(SExplainExecInfo));
QW_ERR_RET(qGetExplainExecInfo(taskHandle, execInfoList));
if (NULL == execInfoList) {
QW_ERR_JRET(terrno);
}
QW_ERR_JRET(qGetExplainExecInfo(taskHandle, execInfoList));
if (ctx->localExec) {
SExplainLocalRsp localRsp = {0};
localRsp.rsp.numOfPlans = taosArrayGetSize(execInfoList);
SExplainExecInfo *pExec = taosMemoryCalloc(localRsp.rsp.numOfPlans, sizeof(SExplainExecInfo));
memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo));
if (NULL == pExec) {
QW_ERR_JRET(terrno);
}
(void)memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo));
localRsp.rsp.subplanInfo = pExec;
localRsp.qId = qId;
localRsp.tId = tId;
localRsp.rId = rId;
localRsp.eId = eId;
taosArrayPush(ctx->explainRes, &localRsp);
if (NULL == taosArrayPush(ctx->explainRes, &localRsp)) {
QW_ERR_JRET(terrno);
}
taosArrayDestroy(execInfoList);
execInfoList = NULL;
} else {
SRpcHandleInfo connInfo = ctx->ctrlConnInfo;
connInfo.ahandle = NULL;
int32_t code = qwBuildAndSendExplainRsp(&connInfo, execInfoList);
taosArrayDestroyEx(execInfoList, freeExplainExecItem);
execInfoList = NULL;
QW_ERR_RET(code);
}
return TSDB_CODE_SUCCESS;
_return:
taosArrayDestroyEx(execInfoList, freeExplainExecItem);
return code;
}
@ -503,37 +521,48 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) {
*pParam = &gQwMgmt.param[paramIdx];
}
void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) {
int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) {
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
STbVerInfo tbInfo;
int32_t i = 0;
int32_t code = TSDB_CODE_SUCCESS;
bool tbGet = false;
while (true) {
if (qGetQueryTableSchemaVersion(pTaskInfo, dbFName, tbName, &tbInfo.sversion, &tbInfo.tversion, i) < 0) {
tbGet = false;
code = qGetQueryTableSchemaVersion(pTaskInfo, dbFName, tbName, &tbInfo.sversion, &tbInfo.tversion, i, &tbGet);
if (TSDB_CODE_SUCCESS != code || !tbGet) {
break;
}
if (dbFName[0] && tbName[0]) {
sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName);
(void)sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName);
} else {
tbInfo.tbFName[0] = 0;
}
if (NULL == ctx->tbInfo) {
ctx->tbInfo = taosArrayInit(1, sizeof(tbInfo));
if (NULL == ctx->tbInfo) {
QW_ERR_RET(terrno);
}
}
taosArrayPush(ctx->tbInfo, &tbInfo);
if (NULL == taosArrayPush(ctx->tbInfo, &tbInfo)) {
QW_ERR_RET(terrno);
}
i++;
}
QW_RET(code);
}
void qwCloseRef(void) {
taosWLockLatch(&gQwMgmt.lock);
if (atomic_load_32(&gQwMgmt.qwNum) <= 0 && gQwMgmt.qwRef >= 0) {
taosCloseRef(gQwMgmt.qwRef);
(void)taosCloseRef(gQwMgmt.qwRef); // ignore error
gQwMgmt.qwRef = -1;
}
taosWUnLockLatch(&gQwMgmt.lock);
@ -550,7 +579,7 @@ void qwDestroyImpl(void *pMgmt) {
int32_t schStatusCount = 0;
qDebug("start to destroy qworker, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt);
taosTmrStop(mgmt->hbTimer);
(void)taosTmrStop(mgmt->hbTimer); //ignore error
mgmt->hbTimer = NULL;
taosTmrCleanUp(mgmt->timer);
@ -584,7 +613,7 @@ void qwDestroyImpl(void *pMgmt) {
taosMemoryFree(mgmt);
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
(void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
qwCloseRef();
@ -641,24 +670,33 @@ int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type) {
return pStat->num ? (pStat->total / pStat->num) : 0;
default:
qError("unsupported queue type %d", type);
break;
}
return -1;
}
void qwClearExpiredSch(SQWorker *mgmt, SArray *pExpiredSch) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t num = taosArrayGetSize(pExpiredSch);
for (int32_t i = 0; i < num; ++i) {
uint64_t *sId = taosArrayGet(pExpiredSch, i);
SQWSchStatus *pSch = NULL;
if (qwAcquireScheduler(mgmt, *sId, QW_WRITE, &pSch)) {
if (NULL == sId) {
qError("get the %dth sch failed, code:%x", i, terrno);
break;
}
code = qwAcquireScheduler(mgmt, *sId, QW_WRITE, &pSch);
if (TSDB_CODE_SUCCESS != code) {
qError("acquire sch %" PRIx64 " failed, code:%x", *sId, code);
continue;
}
if (taosHashGetSize(pSch->tasksHash) <= 0) {
qwDestroySchStatus(pSch);
taosHashRemove(mgmt->schHash, sId, sizeof(*sId));
qDebug("sch %" PRIx64 " destroyed", *sId);
code = taosHashRemove(mgmt->schHash, sId, sizeof(*sId));
qDebug("sch %" PRIx64 " destroy result code:%x", *sId, code);
}
qwReleaseScheduler(QW_WRITE, mgmt);

163
source/libs/qworker/src/qworker.c
View File

@ -18,10 +18,11 @@ SQWorkerMgmt gQwMgmt = {
.qwNum = 0,
};
int32_t qwStopAllTasks(SQWorker *mgmt) {
void qwStopAllTasks(SQWorker *mgmt) {
uint64_t qId, tId, sId;
int32_t eId;
int64_t rId = 0;
int32_t code = TSDB_CODE_SUCCESS;
void *pIter = taosHashIterate(mgmt->ctxHash, NULL);
while (pIter) {
@ -44,22 +45,29 @@ int32_t qwStopAllTasks(SQWorker *mgmt) {
}
if (QW_QUERY_RUNNING(ctx)) {
qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED);
QW_TASK_DLOG_E("task running, async killed");
code = qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED);
if (TSDB_CODE_SUCCESS != code) {
QW_TASK_ELOG("task running, async kill failed, error: %x", code);
} else {
QW_TASK_DLOG_E("task running, async killed");
}
} else if (QW_FETCH_RUNNING(ctx)) {
QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED);
QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP);
QW_TASK_DLOG_E("task fetching, update drop received");
} else {
qwDropTask(QW_FPARAMS());
code = qwDropTask(QW_FPARAMS());
if (TSDB_CODE_SUCCESS != code) {
QW_TASK_ELOG("task drop failed, error: %x", code);
} else {
QW_TASK_DLOG_E("task dropped");
}
}
QW_UNLOCK(QW_WRITE, &ctx->lock);
pIter = taosHashIterate(mgmt->ctxHash, pIter);
}
return TSDB_CODE_SUCCESS;
}
int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
@ -111,7 +119,7 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
int32_t qwSendQueryRsp(QW_FPARAMS_DEF, int32_t msgType, SQWTaskCtx *ctx, int32_t rspCode, bool quickRsp) {
if ((!quickRsp) || QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy) {
if (!ctx->localExec) {
qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx);
QW_ERR_RET(qwBuildAndSendQueryRsp(msgType, &ctx->ctrlConnInfo, rspCode, ctx));
QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, rspCode, tstrerror(rspCode));
}
@ -140,6 +148,10 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
}
SArray *pResList = taosArrayInit(4, POINTER_BYTES);
if (NULL == pResList) {
QW_ERR_RET(terrno);
}
while (true) {
QW_TASK_DLOG("start to execTask, loopIdx:%d", i++);
@ -165,6 +177,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
size_t numOfResBlock = taosArrayGetSize(pResList);
for (int32_t j = 0; j < numOfResBlock; ++j) {
SSDataBlock *pRes = taosArrayGetP(pResList, j);
if (NULL == pRes) {
QW_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SInputData inputData = {.pData = pRes};
code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue);
@ -226,7 +241,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
}
_return:
taosArrayDestroy(pResList);
QW_RET(code);
}
@ -241,7 +258,8 @@ bool qwTaskNotInExec(SQWTaskCtx *ctx) {
int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) {
int32_t taskNum = 0;
int32_t code = TSDB_CODE_SUCCESS;
hbInfo->connInfo = sch->hbConnInfo;
hbInfo->rsp.epId = sch->hbEpId;
@ -272,7 +290,11 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo)
status.status = taskStatus->status;
status.refId = taskStatus->refId;
taosArrayPush(hbInfo->rsp.taskStatus, &status);
if (NULL == taosArrayPush(hbInfo->rsp.taskStatus, &status)) {
taosHashCancelIterate(sch->tasksHash, pIter);
code = terrno;
break;
}
++i;
pIter = taosHashIterate(sch->tasksHash, pIter);
@ -280,7 +302,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo)
QW_UNLOCK(QW_READ, &sch->tasksLock);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, int32_t *pRawDataLen, void **rspMsg,
@ -320,7 +342,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
pOutput->numOfRows);
if (!ctx->dynamicTask) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask);
QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask));
}
if (NULL == pRsp) {
@ -375,7 +397,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen,
if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) {
QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %" PRId64, pOutput->numOfBlocks,
pOutput->numOfRows);
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask);
QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask));
break;
}
@ -430,8 +452,8 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes
pRes->skey = pDelRes->skey;
pRes->ekey = pDelRes->ekey;
pRes->affectedRows = pDelRes->affectedRows;
strcpy(pRes->tableFName, pDelRes->tableName);
strcpy(pRes->tsColName, pDelRes->tsColName);
TAOS_STRCPY(pRes->tableFName, pDelRes->tableName);
TAOS_STRCPY(pRes->tsColName, pDelRes->tsColName);
taosMemoryFree(output.pData);
return TSDB_CODE_SUCCESS;
@ -464,7 +486,7 @@ int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg, int32
qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
QW_ERR_RET(qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code));
rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
@ -650,7 +672,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
_return:
if (TSDB_CODE_SUCCESS == code && QW_PHASE_POST_QUERY == phase) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC, ctx->dynamicTask);
code = qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC, ctx->dynamicTask);
ctx->queryGotData = true;
}
@ -660,7 +682,10 @@ _return:
qwDbgSimulateRedirect(&qwMsg, ctx, &rsped);
qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped);
if (!rsped) {
qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false);
int32_t newCode = qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false);
if (TSDB_CODE_SUCCESS != newCode && TSDB_CODE_SUCCESS == code) {
code = newCode;
}
}
}
@ -672,7 +697,7 @@ _return:
}
if (code) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask);
(void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); // already in error, ignore new error
}
QW_UNLOCK(QW_WRITE, &ctx->lock);
@ -687,11 +712,11 @@ _return:
int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF) {
QW_ERR_RET(qwDropTask(QW_FPARAMS()));
QW_RET(TSDB_CODE_SUCCESS);
return TSDB_CODE_SUCCESS;
}
int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
int32_t code = 0;
int32_t code = TSDB_CODE_SUCCESS;
SQWTaskCtx *ctx = NULL;
QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo));
@ -706,7 +731,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT));
qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true);
QW_ERR_JRET(qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true));
_return:
@ -715,7 +740,7 @@ _return:
qwReleaseTaskCtx(mgmt, ctx);
}
QW_RET(TSDB_CODE_SUCCESS);
return TSDB_CODE_SUCCESS;
}
int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
@ -761,7 +786,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
atomic_store_ptr(&ctx->taskHandle, pTaskInfo);
atomic_store_ptr(&ctx->sinkHandle, sinkHandle);
qwSaveTbVersionInfo(pTaskInfo, ctx);
QW_ERR_JRET(qwSaveTbVersionInfo(pTaskInfo, ctx));
if (!ctx->dynamicTask) {
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL));
@ -778,7 +803,7 @@ _return:
input.msgType = qwMsg->msgType;
code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL);
qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code);
QW_ERR_RET(qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code));
QW_RET(TSDB_CODE_SUCCESS);
}
@ -829,7 +854,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
QW_ERR_JRET(qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code));
rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code,
@ -851,9 +876,14 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
rsp = NULL;
qwMsg->connInfo = ctx->dataConnInfo;
qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, NULL, 0, code);
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
0);
code = qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, NULL, 0, code);
if (TSDB_CODE_SUCCESS != code) {
QW_TASK_ELOG("fetch rsp send fail, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
0);
} else {
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
0);
}
}
QW_LOCK(QW_WRITE, &ctx->lock);
@ -869,7 +899,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
} while (true);
input.code = code;
qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL);
QW_ERR_RET(qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL));
QW_RET(TSDB_CODE_SUCCESS);
}
@ -922,7 +952,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
} else if (QW_QUERY_RUNNING(ctx)) {
atomic_store_8((int8_t *)&ctx->queryContinue, 1);
} else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) {
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC, ctx->dynamicTask);
QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC, ctx->dynamicTask));
atomic_store_8((int8_t *)&ctx->queryInQueue, 1);
QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), &qwMsg->connInfo));
@ -952,9 +982,14 @@ _return:
}
if (!rsped) {
qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
QW_TASK_DLOG("fetch rsp send, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1),
qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
code = qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
if (TSDB_CODE_SUCCESS != code) {
QW_TASK_ELOG("fetch rsp send fail, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1),
qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
} else {
QW_TASK_DLOG("fetch rsp send, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1),
qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
}
} else {
qwFreeFetchRsp(rsp);
rsp = NULL;
@ -985,7 +1020,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
if (QW_QUERY_RUNNING(ctx)) {
QW_ERR_JRET(qwKillTaskHandle(ctx, TSDB_CODE_TSC_QUERY_CANCELLED));
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP, ctx->dynamicTask);
QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP, ctx->dynamicTask));
} else {
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
dropped = true;
@ -1001,7 +1036,7 @@ _return:
if (code) {
if (ctx) {
QW_UPDATE_RSP_CODE(ctx, code);
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask);
(void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); // task already failed, no more error handling
} else {
tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER);
}
@ -1035,7 +1070,7 @@ int32_t qwProcessNotify(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
if (QW_QUERY_RUNNING(ctx)) {
QW_ERR_JRET(qwKillTaskHandle(ctx, TSDB_CODE_TSC_QUERY_CANCELLED));
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask);
QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask));
}
switch (qwMsg->msgType) {
@ -1055,7 +1090,7 @@ _return:
if (code) {
if (ctx) {
QW_UPDATE_RSP_CODE(ctx, code);
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask);
(void)qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); // task already failed, no more error handling
}
}
@ -1091,8 +1126,8 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
sch->hbConnInfo.handle = NULL;
}
memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo));
memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId));
(void)memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo));
(void)memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId));
QW_UNLOCK(QW_WRITE, &sch->hbConnLock);
@ -1103,8 +1138,8 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
_return:
memcpy(&rsp.epId, &req->epId, sizeof(req->epId));
qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code);
(void)memcpy(&rsp.epId, &req->epId, sizeof(req->epId));
code = qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code);
if (code) {
tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER);
@ -1125,7 +1160,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
int64_t refId = hbParam->refId;
SQWorker *mgmt = qwAcquire(refId);
if (NULL == mgmt) {
QW_DLOG("qwAcquire %" PRIx64 "failed", refId);
QW_DLOG("qwAcquire %" PRIx64 "failed, code:0x%x", refId, terrno);
return;
}
@ -1137,7 +1172,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
qwDbgDumpMgmtInfo(mgmt);
if (gQWDebug.forceStop) {
(void)qwStopAllTasks(mgmt);
qwStopAllTasks(mgmt);
}
QW_LOCK(QW_READ, &mgmt->schLock);
@ -1145,8 +1180,8 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
int32_t schNum = taosHashGetSize(mgmt->schHash);
if (schNum <= 0) {
QW_UNLOCK(QW_READ, &mgmt->schLock);
taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer);
qwRelease(refId);
(void)taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); // ignore error
(void)qwRelease(refId); // ignore error
return;
}
@ -1156,9 +1191,9 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
QW_UNLOCK(QW_READ, &mgmt->schLock);
taosMemoryFree(rspList);
taosArrayDestroy(pExpiredSch);
QW_ELOG("calloc %d SQWHbInfo failed", schNum);
taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer);
qwRelease(refId);
QW_ELOG("calloc %d SQWHbInfo failed, code:%x", schNum, terrno);
(void)taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); // ignore error
(void)qwRelease(refId); // ignore error
return;
}
@ -1174,7 +1209,11 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
if (sch1->hbBrokenTs > 0 && ((currentMs - sch1->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) &&
taosHashGetSize(sch1->tasksHash) <= 0) {
taosArrayPush(pExpiredSch, sId);
if (NULL == taosArrayPush(pExpiredSch, sId)) {
QW_ELOG("add sId 0x%" PRIx64 " to expiredSch failed, code:%x", *sId, terrno);
taosHashCancelIterate(mgmt->schHash, pIter);
break;
}
}
pIter = taosHashIterate(mgmt->schHash, pIter);
@ -1196,7 +1235,7 @@ _return:
QW_UNLOCK(QW_READ, &mgmt->schLock);
for (int32_t j = 0; j < i; ++j) {
qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code);
(void)qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code); // ignore error
/*QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),*/
/*(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));*/
tFreeSSchedulerHbRsp(&rspList[j].rsp);
@ -1209,8 +1248,8 @@ _return:
taosMemoryFreeClear(rspList);
taosArrayDestroy(pExpiredSch);
taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer);
qwRelease(refId);
(void)taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); // ignore error
(void)qwRelease(refId); // ignore error
}
int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) {
@ -1260,19 +1299,19 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S
int32_t qwNum = atomic_add_fetch_32(&gQwMgmt.qwNum, 1);
if (1 == qwNum) {
memset(gQwMgmt.param, 0, sizeof(gQwMgmt.param));
TAOS_MEMSET(gQwMgmt.param, 0, sizeof(gQwMgmt.param));
}
int32_t code = qwOpenRef();
if (code) {
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
(void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
QW_RET(code);
}
SQWorker *mgmt = taosMemoryCalloc(1, sizeof(SQWorker));
if (NULL == mgmt) {
qError("calloc %d failed", (int32_t)sizeof(SQWorker));
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
(void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
QW_RET(TSDB_CODE_OUT_OF_MEMORY);
}
@ -1306,7 +1345,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S
if (pMsgCb) {
mgmt->msgCb = *pMsgCb;
} else {
memset(&mgmt->msgCb, 0, sizeof(mgmt->msgCb));
TAOS_MEMSET(&mgmt->msgCb, 0, sizeof(mgmt->msgCb));
}
mgmt->refId = taosAddRef(gQwMgmt.qwRef, mgmt);
@ -1333,14 +1372,14 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S
_return:
if (mgmt->refId >= 0) {
qwRelease(mgmt->refId);
(void)qwRelease(mgmt->refId); // ignore error
} else {
taosHashCleanup(mgmt->schHash);
taosHashCleanup(mgmt->ctxHash);
taosTmrCleanUp(mgmt->timer);
taosMemoryFreeClear(mgmt);
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
(void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
}
QW_RET(code);
@ -1353,7 +1392,7 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) {
atomic_store_8(&mgmt->nodeStopped, 1);
(void)qwStopAllTasks(mgmt);
qwStopAllTasks(mgmt);
}
void qWorkerDestroy(void **qWorkerMgmt) {
@ -1383,7 +1422,7 @@ int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pSt
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
SDataSinkStat sinkStat = {0};
dsDataSinkGetCacheSize(&sinkStat);
QW_ERR_RET(dsDataSinkGetCacheSize(&sinkStat));
pStat->cacheDataSize = sinkStat.cachedSize;
pStat->queryProcessed = QW_STAT_GET(mgmt->stat.msgStat.queryProcessed);
@ -1427,6 +1466,10 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64
ctx->explainRes = explainRes;
rHandle.pMsgCb = taosMemoryCalloc(1, sizeof(SMsgCb));
if (NULL == rHandle.pMsgCb) {
QW_ERR_JRET(terrno);
}
rHandle.pMsgCb->clientRpc = qwMsg->connInfo.handle;
code = qCreateExecTask(&rHandle, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH);

181
source/libs/qworker/test/qworkerTests.cpp
View File

@ -130,30 +130,32 @@ void qwtBuildFetchReqMsg(SResFetchReq *fetchMsg, SRpcMsg *fetchRpc) {
fetchRpc->contLen = sizeof(SResFetchReq);
}
void qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) {
int qwtBuildDropReqMsg(STaskDropReq *dropMsg, SRpcMsg *dropRpc) {
dropMsg->sId = 1;
dropMsg->queryId = atomic_load_64(&qwtTestQueryId);
dropMsg->taskId = 1;
int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, dropMsg);
if (msgSize < 0) {
return;
return terrno;
}
char *msg = (char*)taosMemoryCalloc(1, msgSize);
if (NULL == msg) {
return;
return terrno;
}
if (tSerializeSTaskDropReq(msg, msgSize, dropMsg) < 0) {
taosMemoryFree(msg);
return;
return terrno;
}
dropRpc->msgType = TDMT_SCH_DROP_TASK;
dropRpc->pCont = msg;
dropRpc->contLen = msgSize;
return TSDB_CODE_SUCCESS;
}
int32_t qwtStringToPlan(const char *str, SSubplan **subplan) {
@ -164,7 +166,11 @@ int32_t qwtStringToPlan(const char *str, SSubplan **subplan) {
int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) {
taosWLockLatch(&qwtTestFetchQueueLock);
struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg));
memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
if (NULL == newMsg) {
printf("malloc failed");
assert(0);
}
(void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg;
if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) {
qwtTestFetchQueueWIdx = 0;
@ -178,7 +184,10 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) {
}
taosWUnLockLatch(&qwtTestFetchQueueLock);
tsem_post(&qwtTestFetchSem);
if (tsem_post(&qwtTestFetchSem) < 0) {
printf("tsem_post failed, errno:%d", errno);
assert(0);
}
return 0;
}
@ -186,7 +195,11 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) {
int32_t qwtPutReqToQueue(void *node, EQueueType qtype, struct SRpcMsg *pMsg) {
taosWLockLatch(&qwtTestQueryQueueLock);
struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg));
memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
if (NULL == newMsg) {
printf("malloc failed");
assert(0);
}
(void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg;
if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) {
qwtTestQueryQueueWIdx = 0;
@ -200,22 +213,34 @@ int32_t qwtPutReqToQueue(void *node, EQueueType qtype, struct SRpcMsg *pMsg) {
}
taosWUnLockLatch(&qwtTestQueryQueueLock);
tsem_post(&qwtTestQuerySem);
if (tsem_post(&qwtTestQuerySem) < 0) {
printf("tsem_post failed, errno:%d", errno);
assert(0);
}
return 0;
}
void qwtSendReqToDnode(void *pVnode, struct SEpSet *epSet, struct SRpcMsg *pReq) {}
void qwtRpcSendResponse(const SRpcMsg *pRsp) {
int qwtRpcSendResponse(const SRpcMsg *pRsp) {
int32_t code = 0;
switch (pRsp->msgType) {
case TDMT_SCH_QUERY_RSP:
case TDMT_SCH_MERGE_QUERY_RSP: {
SQueryTableRsp *rsp = (SQueryTableRsp *)pRsp->pCont;
if (pRsp->code) {
qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc);
qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc);
code = qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc);
if (code) {
assert(0);
return code;
}
code = qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc);
if (code) {
assert(0);
return code;
}
}
rpcFreeCont(rsp);
@ -227,13 +252,25 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) {
if (0 == pRsp->code && 0 == rsp->completed) {
qwtBuildFetchReqMsg(&qwtfetchMsg, &qwtfetchRpc);
qwtPutReqToFetchQueue((void *)0x1, &qwtfetchRpc);
code = qwtPutReqToFetchQueue((void *)0x1, &qwtfetchRpc);
if (code) {
assert(0);
return code;
}
rpcFreeCont(rsp);
return;
return code;
}
qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc);
qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc);
code = qwtBuildDropReqMsg(&qwtdropMsg, &qwtdropRpc);
if (code) {
assert(0);
return code;
}
code = qwtPutReqToFetchQueue((void *)0x1, &qwtdropRpc);
if (code) {
assert(0);
return code;
}
rpcFreeCont(rsp);
break;
@ -245,9 +282,11 @@ void qwtRpcSendResponse(const SRpcMsg *pRsp) {
qwtTestCaseFinished = true;
break;
}
default:
break;
}
return;
return code;
}
int32_t qwtCreateExecTask(void *tsdb, int32_t vgId, uint64_t taskId, struct SSubplan *pPlan, qTaskInfo_t *pTaskInfo,
@ -292,6 +331,9 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock **pRes, uint64_t *useconds) {
if (endExec) {
*pRes = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
if (NULL == *pRes) {
return terrno;
}
(*pRes)->info.rows = taosRand() % 1000 + 1;
} else {
*pRes = NULL;
@ -631,7 +673,7 @@ void *queryThread(void *param) {
while (!qwtTestStop) {
qwtBuildQueryReqMsg(&queryRpc);
qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc, 0);
(void)qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc, 0); // ignore error
if (qwtTestEnableSleep) {
taosUsleep(taosRand() % 5);
}
@ -653,7 +695,7 @@ void *fetchThread(void *param) {
while (!qwtTestStop) {
qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc);
code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc, 0);
(void)qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc, 0); // ignore error
if (qwtTestEnableSleep) {
taosUsleep(taosRand() % 5);
}
@ -674,8 +716,11 @@ void *dropThread(void *param) {
STaskDropReq dropMsg = {0};
while (!qwtTestStop) {
qwtBuildDropReqMsg(&dropMsg, &dropRpc);
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc, 0);
if (0 != qwtBuildDropReqMsg(&dropMsg, &dropRpc)) {
break;
}
(void)qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc, 0); // ignore error
if (qwtTestEnableSleep) {
taosUsleep(taosRand() % 5);
}
@ -700,7 +745,7 @@ void *qwtclientThread(void *param) {
qwtTestCaseFinished = false;
qwtBuildQueryReqMsg(&queryRpc);
qwtPutReqToQueue((void *)0x1, QUERY_QUEUE, &queryRpc);
(void)qwtPutReqToQueue((void *)0x1, QUERY_QUEUE, &queryRpc); //ignore error
while (!qwtTestCaseFinished) {
taosUsleep(1);
@ -752,9 +797,9 @@ void *queryQueueThread(void *param) {
}
if (TDMT_SCH_QUERY == queryRpc->msgType) {
qWorkerProcessQueryMsg(mockPointer, mgmt, queryRpc, 0);
(void)qWorkerProcessQueryMsg(mockPointer, mgmt, queryRpc, 0); //ignore error
} else if (TDMT_SCH_QUERY_CONTINUE == queryRpc->msgType) {
qWorkerProcessCQueryMsg(mockPointer, mgmt, queryRpc, 0);
(void)qWorkerProcessCQueryMsg(mockPointer, mgmt, queryRpc, 0); //ignore error
} else {
printf("unknown msg in query queue, type:%d\n", queryRpc->msgType);
assert(0);
@ -810,16 +855,16 @@ void *fetchQueueThread(void *param) {
switch (fetchRpc->msgType) {
case TDMT_SCH_FETCH:
case TDMT_SCH_MERGE_FETCH:
qWorkerProcessFetchMsg(mockPointer, mgmt, fetchRpc, 0);
(void)qWorkerProcessFetchMsg(mockPointer, mgmt, fetchRpc, 0); //ignore error
break;
case TDMT_SCH_CANCEL_TASK:
//qWorkerProcessCancelMsg(mockPointer, mgmt, fetchRpc, 0);
break;
case TDMT_SCH_DROP_TASK:
qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc, 0);
(void)qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc, 0); //ignore error
break;
case TDMT_SCH_TASK_NOTIFY:
qWorkerProcessNotifyMsg(mockPointer, mgmt, fetchRpc, 0);
(void)qWorkerProcessNotifyMsg(mockPointer, mgmt, fetchRpc, 0); //ignore error
break;
default:
printf("unknown msg type:%d in fetch queue", fetchRpc->msgType);
@ -853,7 +898,7 @@ TEST(seqTest, normalCase) {
qwtBuildQueryReqMsg(&queryRpc);
qwtBuildFetchReqMsg(&qwtfetchMsg, &fetchRpc);
qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc);
(void)qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); //ignore error
stubSetStringToPlan();
stubSetRpcSendResponse();
@ -898,7 +943,7 @@ TEST(seqTest, cancelFirst) {
qwtInitLogFile();
qwtBuildQueryReqMsg(&queryRpc);
qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc);
(void)qwtBuildDropReqMsg(&qwtdropMsg, &dropRpc); //ignore error
stubSetStringToPlan();
stubSetRpcSendResponse();
@ -954,7 +999,7 @@ TEST(seqTest, randCase) {
if (r >= 0 && r < maxr / 5) {
printf("Query,%d\n", t++);
qwtBuildQueryReqMsg(&queryRpc);
code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc, 0);
(void)qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc, 0); //ignore error
} else if (r >= maxr / 5 && r < maxr * 2 / 5) {
// printf("Ready,%d\n", t++);
// qwtBuildReadyReqMsg(&readyMsg, &readyRpc);
@ -965,14 +1010,14 @@ TEST(seqTest, randCase) {
} else if (r >= maxr * 2 / 5 && r < maxr * 3 / 5) {
printf("Fetch,%d\n", t++);
qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc);
code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc, 0);
(void)qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc, 0); //ignore error
if (qwtTestEnableSleep) {
taosUsleep(1);
}
} else if (r >= maxr * 3 / 5 && r < maxr * 4 / 5) {
printf("Drop,%d\n", t++);
qwtBuildDropReqMsg(&dropMsg, &dropRpc);
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc, 0);
(void)qwtBuildDropReqMsg(&dropMsg, &dropRpc); //ignore error
(void)qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc, 0); //ignore error
if (qwtTestEnableSleep) {
taosUsleep(1);
}
@ -1018,14 +1063,14 @@ TEST(seqTest, multithreadRand) {
ASSERT_EQ(code, 0);
TdThreadAttr thattr;
taosThreadAttrInit(&thattr);
(void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5, t6;
taosThreadCreate(&(t1), &thattr, queryThread, mgmt);
// taosThreadCreate(&(t2), &thattr, readyThread, NULL);
taosThreadCreate(&(t3), &thattr, fetchThread, NULL);
taosThreadCreate(&(t4), &thattr, dropThread, NULL);
taosThreadCreate(&(t6), &thattr, fetchQueueThread, mgmt);
(void)taosThreadCreate(&(t1), &thattr, queryThread, mgmt); //ignore error
// (void)taosThreadCreate(&(t2), &thattr, readyThread, NULL); //ignore error
(void)taosThreadCreate(&(t3), &thattr, fetchThread, NULL); //ignore error
(void)taosThreadCreate(&(t4), &thattr, dropThread, NULL); //ignore error
(void)taosThreadCreate(&(t6), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) {
if (qwtTestDeadLoop) {
@ -1083,16 +1128,16 @@ TEST(rcTest, shortExecshortDelay) {
qwtTestMaxExecTaskUsec = 0;
qwtTestReqMaxDelayUsec = 0;
tsem_init(&qwtTestQuerySem, 0, 0);
tsem_init(&qwtTestFetchSem, 0, 0);
(void)tsem_init(&qwtTestQuerySem, 0, 0); //ignore error
(void)tsem_init(&qwtTestFetchSem, 0, 0); //ignore error
TdThreadAttr thattr;
taosThreadAttrInit(&thattr);
(void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5;
taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt);
taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt);
taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt);
(void)taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); //ignore error
(void)taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); //ignore error
(void)taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) {
if (qwtTestDeadLoop) {
@ -1114,8 +1159,8 @@ TEST(rcTest, shortExecshortDelay) {
if (qwtTestCaseFinished) {
if (qwtTestQuitThreadNum < 3) {
tsem_post(&qwtTestQuerySem);
tsem_post(&qwtTestFetchSem);
(void)tsem_post(&qwtTestQuerySem); //ignore error
(void)tsem_post(&qwtTestFetchSem); //ignore error
taosUsleep(10);
}
@ -1166,16 +1211,16 @@ TEST(rcTest, longExecshortDelay) {
qwtTestMaxExecTaskUsec = 1000000;
qwtTestReqMaxDelayUsec = 0;
tsem_init(&qwtTestQuerySem, 0, 0);
tsem_init(&qwtTestFetchSem, 0, 0);
(void)tsem_init(&qwtTestQuerySem, 0, 0); //ignore error
(void)tsem_init(&qwtTestFetchSem, 0, 0); //ignore error
TdThreadAttr thattr;
taosThreadAttrInit(&thattr);
(void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5;
taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt);
taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt);
taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt);
(void)taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); //ignore error
(void)taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); //ignore error
(void)taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) {
if (qwtTestDeadLoop) {
@ -1197,8 +1242,8 @@ TEST(rcTest, longExecshortDelay) {
if (qwtTestCaseFinished) {
if (qwtTestQuitThreadNum < 3) {
tsem_post(&qwtTestQuerySem);
tsem_post(&qwtTestFetchSem);
(void)tsem_post(&qwtTestQuerySem); //ignore error
(void)tsem_post(&qwtTestFetchSem); //ignore error
taosUsleep(10);
}
@ -1249,16 +1294,16 @@ TEST(rcTest, shortExeclongDelay) {
qwtTestMaxExecTaskUsec = 0;
qwtTestReqMaxDelayUsec = 1000000;
tsem_init(&qwtTestQuerySem, 0, 0);
tsem_init(&qwtTestFetchSem, 0, 0);
(void)tsem_init(&qwtTestQuerySem, 0, 0); //ignore error
(void)tsem_init(&qwtTestFetchSem, 0, 0); //ignore error
TdThreadAttr thattr;
taosThreadAttrInit(&thattr);
(void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5;
taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt);
taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt);
taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt);
(void)taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); //ignore error
(void)taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); //ignore error
(void)taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) {
if (qwtTestDeadLoop) {
@ -1280,8 +1325,8 @@ TEST(rcTest, shortExeclongDelay) {
if (qwtTestCaseFinished) {
if (qwtTestQuitThreadNum < 3) {
tsem_post(&qwtTestQuerySem);
tsem_post(&qwtTestFetchSem);
(void)tsem_post(&qwtTestQuerySem); //ignore error
(void)tsem_post(&qwtTestFetchSem); //ignore error
taosUsleep(10);
}
@ -1327,16 +1372,16 @@ TEST(rcTest, dropTest) {
code = qWorkerInit(NODE_TYPE_VNODE, 1, &mgmt, &msgCb);
ASSERT_EQ(code, 0);
tsem_init(&qwtTestQuerySem, 0, 0);
tsem_init(&qwtTestFetchSem, 0, 0);
(void)tsem_init(&qwtTestQuerySem, 0, 0); //ignore error
(void)tsem_init(&qwtTestFetchSem, 0, 0); //ignore error
TdThreadAttr thattr;
taosThreadAttrInit(&thattr);
(void)taosThreadAttrInit(&thattr); //ignore error
TdThread t1, t2, t3, t4, t5;
taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt);
taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt);
taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt);
(void)taosThreadCreate(&(t1), &thattr, qwtclientThread, mgmt); //ignore error
(void)taosThreadCreate(&(t2), &thattr, queryQueueThread, mgmt); //ignore error
(void)taosThreadCreate(&(t3), &thattr, fetchQueueThread, mgmt); //ignore error
while (true) {
if (qwtTestDeadLoop) {

6
source/libs/scalar/src/scalar.c
View File

@ -1654,6 +1654,12 @@ static int32_t sclGetCompOperatorResType(SOperatorNode *pOp) {
(!IS_STR_DATA_TYPE(rdt.type) && (rdt.type != TSDB_DATA_TYPE_NULL))) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
if (nodesIsMatchRegularOp(pOp)) {
SValueNode* node = (SValueNode*)(pOp->pRight);
if(checkRegexPattern(node->literal) != TSDB_CODE_SUCCESS){
return TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR;
}
}
}
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;

3
source/libs/scalar/src/sclvector.c
View File

@ -1675,10 +1675,8 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
pRes[i] = false;
continue;
}
char *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
char *pRightData = colDataGetData(pRight->columnData, rightIndex);
pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData);
if (pRes[i]) {
++num;
@ -1714,7 +1712,6 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
if (!pLeftData || !pRightData) {
result = false;
}
if (!result) {
colDataSetInt8(pOut->columnData, i, (int8_t *)&result);
} else {

7
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -43,6 +43,7 @@
#include "tglobal.h"
#include "tlog.h"
#include "tvariant.h"
#include "tcompare.h"
#define _DEBUG_PRINT_ 0
@ -52,6 +53,12 @@
#define PRINTF(...)
#endif
class constantTest {
public:
constantTest() { InitRegexCache(); }
~constantTest() { DestroyRegexCache(); }
};
static constantTest test;
namespace {
SColumnInfo createColumnInfo(int32_t colId, int32_t type, int32_t bytes) {

4
source/libs/scheduler/inc/schInt.h
View File

@ -583,7 +583,7 @@ char *schGetOpStr(SCH_OP_TYPE type);
int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync);
int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq);
int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq);
int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes);
void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes);
int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet);
int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode);
void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode);
@ -606,7 +606,7 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask);
void schDropTaskInHashList(SSchJob *pJob, SHashObj *list);
int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask);
int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level);
int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask);
void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask);
int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel);
int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask);
void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode);

12
source/libs/scheduler/src/schDbg.c
View File

@ -16,17 +16,5 @@
#include "query.h"
#include "schInt.h"
tsem_t schdRspSem;
SSchDebug gSCHDebug = {0};
void schdExecCallback(SExecResult* pResult, void* param, int32_t code) {
if (code) {
pResult->code = code;
}
*(SExecResult*)param = *pResult;
taosMemoryFree(pResult);
tsem_post(&schdRspSem);
}

13
source/libs/scheduler/src/schFlowCtrl.c
View File

@ -50,6 +50,10 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks);
for (int32_t i = 0; i < taskNum; ++i) {
SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i);
if (NULL == pTask) {
SCH_JOB_DLOG("fail to get the %dth task", i);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
sum += pTask->plan->execNodeStat.tableNum;
}
@ -214,6 +218,10 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
for (int32_t i = 0; i < taskNum; ++i) {
pTask = *(SSchTask **)taosArrayGet(ctrl->taskList, i);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task", i);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) {
@ -243,6 +251,11 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
if (i < (taskNum - 1)) {
SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList);
if (NULL == pLastTask) {
SCH_JOB_ELOG("fail to get the last task, num:%d", (int32_t)taosArrayGetSize(ctrl->taskList));
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (remainNum < pLastTask->plan->execNodeStat.tableNum) {
SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d, smallestInList:%d",
ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum);

160
source/libs/scheduler/src/schJob.c
View File

@ -47,6 +47,7 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) {
return;
_return:
SCH_JOB_DLOG("job errCode updated to %s", tstrerror(errCode));
}
@ -166,9 +167,18 @@ _return:
int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t i = 0; i < pJob->levelNum; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, pJob->levelNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
for (int32_t m = 0; m < pLevel->taskNum; ++m) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, m);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum: %d", m, pLevel->level, pLevel->taskNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSubplan *pPlan = pTask->plan;
int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0;
int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0;
@ -188,6 +198,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t n = 0; n < childNum; ++n) {
SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n);
if (NULL == child) {
SCH_JOB_ELOG("fail to get the %dth child subplan, childNum: %d", n, childNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES);
if (NULL == childTask || NULL == *childTask) {
SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
@ -222,6 +237,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t n = 0; n < parentNum; ++n) {
SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n);
if (NULL == parent) {
SCH_JOB_ELOG("fail to get the %dth parent subplan, parentNum: %d", n, parentNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES);
if (NULL == parentTask || NULL == *parentTask) {
SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
@ -241,6 +261,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
}
SSchLevel *pLevel = taosArrayGet(pJob->levels, 0);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get level 0 level, levelNum:%d", (int32_t)taosArrayGetSize(pJob->levels));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (SCH_IS_QUERY_JOB(pJob)) {
if (pLevel->taskNum > 1) {
SCH_JOB_ELOG("invalid query plan, level:0, taskNum:%d", pLevel->taskNum);
@ -248,6 +273,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
}
SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the first task in level 0, taskNum:%d", (int32_t)taosArrayGetSize(pLevel->subTasks));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType || EXPLAIN_MODE_DISABLE != pJob->attr.explainMode) {
pJob->attr.needFetch = true;
}
@ -261,7 +291,9 @@ int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) {
return TSDB_CODE_SUCCESS;
}
taosArrayPush(pJob->dataSrcTasks, &pTask);
if (NULL == taosArrayPush(pJob->dataSrcTasks, &pTask)) {
SCH_ERR_RET(terrno);
}
return TSDB_CODE_SUCCESS;
}
@ -318,6 +350,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
}
pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, levelNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
pLevel->level = i;
plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i);
@ -342,6 +379,10 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
for (int32_t n = 0; n < taskNum; ++n) {
SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n);
if (NULL == plan) {
SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d", n, taskNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_SET_JOB_TYPE(pJob, plan->subplanType);
@ -383,7 +424,7 @@ _return:
SCH_RET(code);
}
int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) {
void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) {
pRes->code = atomic_load_32(&pJob->errCode);
pRes->numOfRows = pJob->resNumOfRows;
@ -395,8 +436,6 @@ int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) {
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
SCH_JOB_DLOG("execRes dumped, code: %s", tstrerror(pRes->code));
return TSDB_CODE_SUCCESS;
}
int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) {
@ -421,10 +460,13 @@ int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) {
if (NULL == *pData) {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp));
if (rsp) {
rsp->completed = 1;
if (NULL == rsp) {
SCH_JOB_ELOG("malloc SRetrieveTableRsp %d failed, code:%x", (int32_t)sizeof(SRetrieveTableRsp), terrno);
SCH_ERR_JRET(terrno);
}
rsp->completed = 1;
*pData = rsp;
SCH_JOB_DLOG("empty res and set query complete, code:%x", code);
}
@ -440,10 +482,13 @@ _return:
int32_t schNotifyUserExecRes(SSchJob *pJob) {
SExecResult *pRes = taosMemoryCalloc(1, sizeof(SExecResult));
if (pRes) {
schDumpJobExecRes(pJob, pRes);
if (NULL == pRes) {
qError("malloc execResult %d failed, error: %x", (int32_t)sizeof(SExecResult), terrno);
SCH_RET(terrno);
}
schDumpJobExecRes(pJob, pRes);
SCH_JOB_DLOG("sch start to invoke exec cb, code: %s", tstrerror(pJob->errCode));
(*pJob->userRes.execFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode));
SCH_JOB_DLOG("sch end from exec cb, code: %s", tstrerror(pJob->errCode));
@ -454,7 +499,10 @@ int32_t schNotifyUserExecRes(SSchJob *pJob) {
int32_t schNotifyUserFetchRes(SSchJob *pJob) {
void *pRes = NULL;
schDumpJobFetchRes(pJob, &pRes);
int32_t code = schDumpJobFetchRes(pJob, &pRes);
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_SUCCESS == atomic_load_32(&pJob->errCode)) {
atomic_store_32(&pJob->errCode, code);
}
SCH_JOB_DLOG("sch start to invoke fetch cb, code: %s", tstrerror(pJob->errCode));
(*pJob->userRes.fetchFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode));
@ -478,13 +526,13 @@ void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) {
if (SCH_JOB_IN_SYNC_OP(pJob)) {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
tsem_post(&pJob->rspSem);
(void)tsem_post(&pJob->rspSem); // ignore error
} else if (SCH_JOB_IN_ASYNC_EXEC_OP(pJob)) {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
schNotifyUserExecRes(pJob);
(void)schNotifyUserExecRes(pJob); // ignore error
} else if (SCH_JOB_IN_ASYNC_FETCH_OP(pJob)) {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
schNotifyUserFetchRes(pJob);
(void)schNotifyUserFetchRes(pJob); // ignore error
} else {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
SCH_JOB_ELOG("job not in any operation, status:%s", jobTaskStatusStr(pJob->status));
@ -519,7 +567,8 @@ int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode) {
return TSDB_CODE_SCH_IGNORE_ERROR;
}
schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode);
(void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); // ignore error
return TSDB_CODE_SCH_IGNORE_ERROR;
}
@ -530,7 +579,8 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) {
return TSDB_CODE_SCH_IGNORE_ERROR;
}
schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode);
(void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); // ignore error
return TSDB_CODE_SCH_IGNORE_ERROR;
}
@ -569,12 +619,21 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) {
SSchLevel *pLevel = pTask->level;
int32_t doneNum = atomic_load_32(&pLevel->taskExecDoneNum);
if (doneNum == pLevel->taskNum) {
atomic_sub_fetch_32(&pJob->levelIdx, 1);
(void)atomic_sub_fetch_32(&pJob->levelIdx, 1);
pLevel = taosArrayGet(pJob->levels, pJob->levelIdx);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
for (int32_t i = 0; i < pLevel->taskNum; ++i) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, i);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", i, pLevel->level, pLevel->taskNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (pTask->children && taosArrayGetSize(pTask->children) > 0) {
continue;
}
@ -602,7 +661,11 @@ int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) {
}
}
taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo));
if (NULL == taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo))) {
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
SCH_ERR_RET(terrno);
}
taosArrayDestroy(rsp->tbVerInfo);
pJob->execRes.msgType = TDMT_SCH_QUERY;
@ -629,6 +692,11 @@ int32_t schLaunchJob(SSchJob *pJob) {
SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL));
} else {
SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx);
if (NULL == level) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_ERR_RET(schLaunchLevelTasks(pJob, level));
}
@ -661,10 +729,19 @@ void schFreeJobImpl(void *job) {
int32_t numOfLevels = taosArrayGetSize(pJob->levels);
for (int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels);
continue;
}
int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
for (int32_t j = 0; j < numOfTasks; ++j) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth task, taskNum:%d", j, numOfTasks);
continue;
}
schFreeTask(pJob, pTask);
}
@ -687,12 +764,12 @@ void schFreeJobImpl(void *job) {
destroyQueryExecRes(&pJob->execRes);
qDestroyQueryPlan(pJob->pDag);
nodesReleaseAllocatorWeakRef(pJob->allocatorRefId);
(void)nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); // ignore error
taosMemoryFreeClear(pJob->userRes.execRes);
taosMemoryFreeClear(pJob->fetchRes);
taosMemoryFreeClear(pJob->sql);
tsem_destroy(&pJob->rspSem);
(void)tsem_destroy(&pJob->rspSem); // ignore error
taosMemoryFree(pJob);
int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1);
@ -711,7 +788,7 @@ int32_t schJobFetchRows(SSchJob *pJob) {
if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) {
SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
tsem_wait(&pJob->rspSem);
(void)tsem_wait(&pJob->rspSem); // ignore error
SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes));
}
} else {
@ -739,9 +816,19 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
pJob->conn = *pReq->pConn;
if (pReq->sql) {
pJob->sql = taosStrdup(pReq->sql);
if (NULL == pJob->sql) {
qError("QID:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql);
SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
pJob->pDag = pReq->pDag;
pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId);
if (pReq->allocatorRefId > 0) {
pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId);
if (pJob->allocatorRefId <= 0) {
qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId);
SCH_ERR_JRET(terrno);
}
}
pJob->chkKillFp = pReq->chkKillFp;
pJob->chkKillParam = pReq->chkKillParam;
pJob->userRes.execFp = pReq->execFp;
@ -752,6 +839,10 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId);
} else {
pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL);
if (NULL == pJob->nodeList) {
qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, (int32_t)taosArrayGetSize(pReq->pNodeList));
SCH_ERR_JRET(terrno);
}
}
pJob->taskList = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false,
@ -785,7 +876,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
SCH_ERR_JRET(terrno);
}
atomic_add_fetch_32(&schMgmt.jobNum, 1);
(void)atomic_add_fetch_32(&schMgmt.jobNum, 1);
*pJobId = pJob->refId;
@ -800,7 +891,7 @@ _return:
} else if (pJob->refId < 0) {
schFreeJobImpl(pJob);
} else {
taosRemoveRef(schMgmt.jobRef, pJob->refId);
(void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error
}
SCH_RET(code);
@ -814,7 +905,7 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) {
if (pReq->syncReq) {
SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
tsem_wait(&pJob->rspSem);
(void)tsem_wait(&pJob->rspSem); // ignore error
}
SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId);
@ -845,7 +936,7 @@ int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) {
}
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC);
SCH_ERR_RET(schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC));
}
return TSDB_CODE_SUCCESS;
@ -866,6 +957,10 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) {
int32_t numOfLevels = taosArrayGetSize(pJob->levels);
for (int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
pLevel->taskExecDoneNum = 0;
pLevel->taskLaunchedNum = 0;
@ -873,6 +968,11 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) {
int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
for (int32_t j = 0; j < numOfTasks; ++j) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", j, i, numOfTasks);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_LOCK_TASK(pTask);
code = schChkUpdateRedirectCtx(pJob, pTask, NULL, rspCode);
if (TSDB_CODE_SUCCESS != code) {
@ -973,7 +1073,7 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int
}
if (errCode) {
schHandleJobFailure(pJob, errCode);
(void)schHandleJobFailure(pJob, errCode); // handle internal
}
SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode));
@ -1052,11 +1152,11 @@ void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode) {
}
if (errCode) {
schHandleJobFailure(pJob, errCode);
(void)schHandleJobFailure(pJob, errCode); // ignore error
}
if (pJob) {
schReleaseJob(pJob->refId);
(void)schReleaseJob(pJob->refId); // ignore error
}
}
@ -1091,7 +1191,7 @@ _return:
SCH_UNLOCK_TASK(pTask);
}
if (pJob) {
schReleaseJob(rId);
(void)schReleaseJob(rId); // ignore error
}
SCH_RET(code);

10
source/libs/scheduler/src/schUtil.c
View File

@ -316,18 +316,18 @@ void schFreeRpcCtx(SRpcCtx *pCtx) {
}
}
int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) {
void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) {
*pTask = NULL;
int32_t s = taosHashGetSize(pTaskList);
if (s <= 0) {
return TSDB_CODE_SUCCESS;
return;
}
SSchTask **task = taosHashGet(pTaskList, &taskId, sizeof(taskId));
if (NULL == task || NULL == (*task)) {
return TSDB_CODE_SUCCESS;
return;
}
*pTask = *task;
return TSDB_CODE_SUCCESS;
}

26
source/libs/scheduler/src/scheduler.c
View File

@ -108,14 +108,26 @@ int32_t schedulerGetTasksStatus(int64_t jobId, SArray *pSub) {
for (int32_t i = pJob->levelNum - 1; i >= 0; --i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
qError("failed to get level %d", i);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
for (int32_t m = 0; m < pLevel->taskNum; ++m) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, m);
if (NULL == pTask) {
qError("failed to get task %d, total: %d", m, pLevel->taskNum);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SQuerySubDesc subDesc = {0};
subDesc.tid = pTask->taskId;
strcpy(subDesc.status, jobTaskStatusStr(pTask->status));
TAOS_STRCPY(subDesc.status, jobTaskStatusStr(pTask->status));
taosArrayPush(pSub, &subDesc);
if (NULL == taosArrayPush(pSub, &subDesc)) {
qError("taosArrayPush task %d failed, error: %x, ", m, terrno);
SCH_ERR_JRET(terrno);
}
}
}
@ -141,7 +153,7 @@ int32_t schedulerUpdatePolicy(int32_t policy) {
qDebug("schedule policy updated to %d", schMgmt.cfg.schPolicy);
break;
default:
return TSDB_CODE_TSC_INVALID_INPUT;
SCH_RET(TSDB_CODE_TSC_INVALID_INPUT);
}
return TSDB_CODE_SUCCESS;
@ -159,14 +171,14 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) {
SSchJob *pJob = schAcquireJob(*jobId);
if (NULL == pJob) {
qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId);
qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId);
return;
}
SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode));
schHandleJobDrop(pJob, errCode);
(void)schHandleJobDrop(pJob, errCode); // ignore any error
schReleaseJob(*jobId);
(void)schReleaseJob(*jobId); // ignore error
*jobId = 0;
}
@ -182,7 +194,7 @@ void schedulerDestroy(void) {
if (refId == 0) {
break;
}
taosRemoveRef(schMgmt.jobRef, pJob->refId);
(void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error
pJob = taosIterateRef(schMgmt.jobRef, refId);
}

4
source/libs/stream/src/streamCheckStatus.c
View File

@ -308,7 +308,9 @@ int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) {
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
ASSERT(pInfo->inCheckProcess == 0);
pInfo->pList = taosArrayDestroy(pInfo->pList);
taosArrayDestroy(pInfo->pList);
pInfo->pList = NULL;
if (pInfo->checkRspTmr != NULL) {
/*bool ret = */ taosTmrStop(pInfo->checkRspTmr);
pInfo->checkRspTmr = NULL;

6
source/libs/stream/src/streamMsg.c
View File

@ -438,11 +438,13 @@ void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) {
}
if (pMsg->pUpdateNodes != NULL) {
pMsg->pUpdateNodes = taosArrayDestroy(pMsg->pUpdateNodes);
taosArrayDestroy(pMsg->pUpdateNodes);
pMsg->pUpdateNodes = NULL;
}
if (pMsg->pTaskStatus != NULL) {
pMsg->pTaskStatus = taosArrayDestroy(pMsg->pTaskStatus);
taosArrayDestroy(pMsg->pTaskStatus);
pMsg->pTaskStatus = NULL;
}
pMsg->msgId = -1;

Some files were not shown because too many files have changed in this diff Show More