xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into fix/addErrorToMgmt

This commit is contained in:
Yihao Deng 2024-07-18 00:48:38 +00:00
parent 3369f7ae41 aeb21a536e
commit 71ec35a7da
87 changed files with 3460 additions and 1881 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/common/cos_cp.h
View File

@ -60,8 +60,8 @@ typedef struct {
} SCheckpoint;
int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint);
void cos_cp_close(TdFilePtr fd);
void cos_cp_remove(char const* filepath);
int32_t cos_cp_close(TdFilePtr fd);
int32_t cos_cp_remove(char const* filepath);
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
int32_t cos_cp_dump(SCheckpoint* checkpoint);

2
include/common/tdataformat.h
View File

@ -136,7 +136,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
void tRowGetPrimaryKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2);
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
// SRowIter ================================
int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);

2
include/common/tmisce.h
View File

@ -31,7 +31,7 @@ typedef struct SCorEpSet {
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t len);
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
void epsetAssign(SEpSet* dst, const SEpSet* pSrc);

4
include/common/tmsgcb.h
View File

@ -17,13 +17,13 @@
#define _TD_COMMON_MSG_CB_H_
#include "os.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SRpcMsg SRpcMsg;
typedef struct SEpSet SEpSet;
typedef struct SMgmtWrapper SMgmtWrapper;
typedef struct SRpcHandleInfo SRpcHandleInfo;
@ -46,7 +46,7 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg);
typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
typedef void (*SendRspFp)(SRpcMsg* pMsg);
typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg);
typedef void (*RegisterBrokenLinkArgFp)(struct SRpcMsg* pMsg);
typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
typedef void (*ReportStartup)(const char* name, const char* desc);

2
include/common/tname.h
View File

@ -81,7 +81,7 @@ typedef struct {
// uint64_t uid; // child table uid, may be useful
} RandTableName;
void buildChildTableName(RandTableName* rName);
int32_t buildChildTableName(RandTableName* rName);
#ifdef __cplusplus
}

2
include/libs/audit/audit.h
View File

@ -50,7 +50,7 @@ typedef struct {
int32_t auditInit(const SAuditCfg *pCfg);
void auditCleanup();
void auditSend(SJson *pJson);
int32_t auditSend(SJson *pJson);
void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,
char *detail, int32_t len);
void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,

14
include/libs/executor/storageapi.h
View File

@ -105,7 +105,7 @@ typedef struct SMTbCursor {
} SMTbCursor;
typedef struct SMCtbCursor {
SMeta* pMeta;
struct SMeta* pMeta;
void* pCur;
tb_uid_t suid;
void* pKey;
@ -134,7 +134,7 @@ typedef struct SMetaTableInfo {
} SMetaTableInfo;
typedef struct SSnapContext {
SMeta* pMeta;
struct SMeta* pMeta;
int64_t snapVersion;
void* pCur;
int64_t suid;
@ -178,7 +178,7 @@ typedef struct TsdReader {
int32_t (*tsdNextDataBlock)();
int32_t (*tsdReaderRetrieveBlockSMAInfo)();
SSDataBlock *(*tsdReaderRetrieveDataBlock)();
int32_t (*tsdReaderRetrieveDataBlock)();
void (*tsdReaderReleaseDataBlock)();
@ -381,7 +381,7 @@ typedef struct SStateStore {
void** ppVal, int32_t* pVLen);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
@ -389,12 +389,12 @@ typedef struct SStateStore {
void (*updateInfoDestroy)(SUpdateInfo* pInfo);
void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);

4
include/libs/parser/parser.h
View File

@ -149,10 +149,10 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
void qDestroyBoundColInfo(void* pInfo);
SQuery* smlInitHandle();
int32_t smlInitHandle(SQuery** query);
int32_t smlBuildRow(STableDataCxt* pTableCxt);
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index);
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta);
int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt);
void clearColValArraySml(SArray* pCols);
int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,

4
include/libs/scheduler/scheduler.h
View File

@ -23,8 +23,6 @@ extern "C" {
#include "catalog.h"
#include "planner.h"
extern tsem_t schdRspSem;
typedef struct SQueryProfileSummary {
int64_t startTs; // Object created and added into the message queue
int64_t endTs; // the timestamp when the task is completed
@ -101,8 +99,6 @@ void schedulerFreeJob(int64_t* job, int32_t errCode);
void schedulerDestroy(void);
void schdExecCallback(SExecResult* pResult, void* param, int32_t code);
#ifdef __cplusplus
}
#endif

28
include/libs/stream/tstreamUpdate.h
View File

@ -15,29 +15,29 @@
#ifndef _TSTREAMUPDATE_H_
#define _TSTREAMUPDATE_H_
#include "storageapi.h"
#include "taosdef.h"
#include "tarray.h"
#include "tcommon.h"
#include "tmsg.h"
#include "storageapi.h"
#ifdef __cplusplus
extern "C" {
#endif
SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen);
TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo *pInfo);
void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo);
void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo);
int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo);
int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo);
void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count);
void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count);
bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo* pInfo);
void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo);
void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo);
int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count);
int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count);
bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
#ifdef __cplusplus
}

3
include/libs/sync/sync.h
View File

@ -70,7 +70,6 @@ typedef int64_t SyncIndex;
typedef int64_t SyncTerm;
typedef struct SSyncNode SSyncNode;
typedef struct SWal SWal;
typedef struct SSyncRaftEntry SSyncRaftEntry;
typedef enum {
@ -238,7 +237,7 @@ typedef struct SSyncInfo {
int32_t batchSize;
SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN];
SWal* pWal;
struct SWal* pWal;
SSyncFSM* pFsm;
SMsgCb* msgcb;
int32_t pingMs;

6
include/os/osMath.h
View File

@ -32,9 +32,9 @@ extern "C" {
#define TSWAP(a, b) \
do { \
char *__tmp = (char*)alloca(sizeof(a)); \
memcpy(__tmp, &(a), sizeof(a)); \
memcpy(&(a), &(b), sizeof(a)); \
memcpy(&(b), __tmp, sizeof(a)); \
(void)memcpy(__tmp, &(a), sizeof(a)); \
(void)memcpy(&(a), &(b), sizeof(a)); \
(void)memcpy(&(b), __tmp, sizeof(a)); \
} while (0)
#ifdef WINDOWS

3
include/os/osMemory.h
View File

@ -53,6 +53,9 @@ void taosPrintBackTrace();
void taosMemoryTrim(int32_t size);
void *taosMemoryMallocAlign(uint32_t alignment, int64_t size);
#define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n))
#define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n))
#define taosMemoryFreeClear(ptr) \
do { \
if (ptr) { \

4
include/os/osString.h
View File

@ -55,10 +55,12 @@ typedef enum { M2C = 0, C2M } ConvType;
#define tstrncpy(dst, src, size) \
do { \
strncpy((dst), (src), (size)); \
(void)strncpy((dst), (src), (size)); \
(dst)[(size)-1] = 0; \
} while (0)
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4);
int64_t taosStr2int64(const char *str);

9
include/util/talgo.h
View File

@ -52,7 +52,7 @@ typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void
* @param param
* @param comparFn
*/
void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn);
int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn);
/**
* Non-recursive quick sort.
@ -98,8 +98,8 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size
* @param maxroot: if heap is max root heap
* @return
*/
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char *buf, bool maxroot);
int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char *buf, bool maxroot);
/**
* sort heap to make sure it is a max/min root heap
@ -114,7 +114,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
* @param maxroot: if heap is max root heap
* @return
*/
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, bool maxroot);
int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
bool maxroot);
#ifdef __cplusplus
}

5
include/util/taoserror.h
View File

@ -942,6 +942,11 @@ int32_t taosGetErrSize();
// UTIL
#define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000)
// AUDIT
#define TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON TAOS_DEF_ERROR_CODE(0, 0x6100)
#define TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD TAOS_DEF_ERROR_CODE(0, 0x6101)
#define TSDB_CODE_AUDIT_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0x6102)
#ifdef __cplusplus
}
#endif

2
include/util/tarray.h
View File

@ -251,7 +251,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t
* @return
*/
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
int32_t taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver);

22
include/util/tbloomfilter.h
View File

@ -19,6 +19,8 @@
#include "os.h"
#include "tencode.h"
#include "thash.h"
#include "tlog.h"
#include "tutil.h"
#ifdef __cplusplus
extern "C" {
@ -35,19 +37,19 @@ typedef struct SBloomFilter {
uint64_t size;
_hash_fn_t hashFn1;
_hash_fn_t hashFn2;
void *buffer;
void* buffer;
double errorRate;
} SBloomFilter;
SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate);
int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2);
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2);
void tBloomFilterDestroy(SBloomFilter *pBF);
void tBloomFilterDump(const SBloomFilter *pBF);
bool tBloomFilterIsFull(const SBloomFilter *pBF);
int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder);
SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder);
int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF);
int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2);
int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t h1, uint64_t h2);
void tBloomFilterDestroy(SBloomFilter* pBF);
void tBloomFilterDump(const SBloomFilter* pBF);
bool tBloomFilterIsFull(const SBloomFilter* pBF);
int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder);
int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF);
#ifdef __cplusplus
}

368
include/util/tencode.h
View File

@ -18,6 +18,7 @@
#include "tcoding.h"
#include "tlist.h"
#include "tutil.h"
#ifdef __cplusplus
extern "C" {
@ -46,38 +47,8 @@ typedef struct {
SDecoderNode* dStack;
} SDecoder;
#define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL)
#define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0]
#define tRPut16(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[0];
#define tRPut32(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[0];
#define tRPut64(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[7]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[6]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[5]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[4]; \
((uint8_t*)(PDEST))[4] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[5] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[6] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[7] = ((uint8_t*)(PSRC))[0];
#define tRGet16 tRPut16
#define tRGet32 tRPut32
#define tRGet64 tRPut64
#define TD_CODER_POS(CODER) ((CODER)->pos)
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos)
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos)
#define tEncodeSize(E, S, SIZE, RET) \
do { \
@ -100,6 +71,7 @@ void tEncoderInit(SEncoder* pCoder, uint8_t* data, uint32_t size);
void tEncoderClear(SEncoder* pCoder);
int32_t tStartEncode(SEncoder* pCoder);
void tEndEncode(SEncoder* pCoder);
static int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size);
static int32_t tEncodeU8(SEncoder* pCoder, uint8_t val);
static int32_t tEncodeI8(SEncoder* pCoder, int8_t val);
static int32_t tEncodeU16(SEncoder* pCoder, uint16_t val);
@ -127,6 +99,7 @@ void tDecoderClear(SDecoder* SDecoder);
int32_t tStartDecode(SDecoder* pCoder);
void tEndDecode(SDecoder* pCoder);
static bool tDecodeIsEnd(SDecoder* pCoder);
static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size);
static int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val);
static int32_t tDecodeI8(SDecoder* pCoder, int8_t* val);
static int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val);
@ -149,97 +122,65 @@ static int32_t tDecodeCStr(SDecoder* pCoder, char** val);
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val);
/* ------------------------ IMPL ------------------------ */
#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) return -1; \
tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL)); \
} \
TD_CODER_MOVE_POS(CODER, sizeof(VAL)); \
return 0;
#define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \
while ((VAL) >= ENCODE_LIMIT) { \
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \
} \
\
(VAL) >>= 7; \
TD_CODER_MOVE_POS(CODER, 1); \
} \
\
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TD_CODER_CURRENT(CODER)[0] = (uint8_t)(VAL); \
} \
TD_CODER_MOVE_POS(CODER, 1); \
return 0;
#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS) \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) return -1; \
tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL)); \
TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \
return 0;
#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \
int32_t i = 0; \
*(PVAL) = 0; \
for (;;) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TYPE tval = TD_CODER_CURRENT(CODER)[0]; \
if (tval < ENCODE_LIMIT) { \
*(PVAL) |= (tval << (7 * i)); \
TD_CODER_MOVE_POS(pCoder, 1); \
break; \
} else { \
*(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \
i++; \
TD_CODER_MOVE_POS(pCoder, 1); \
} \
} \
\
return 0;
// 8
static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) {
static FORCE_INLINE int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size) {
if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1;
tPut(uint8_t, TD_CODER_CURRENT(pCoder), val);
if (pCoder->pos + size > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
memcpy(pCoder->data + pCoder->pos, val, size);
}
TD_CODER_MOVE_POS(pCoder, sizeof(val));
pCoder->pos += size;
return 0;
}
static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) {
if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1;
tPut(int8_t, TD_CODER_CURRENT(pCoder), val);
}
TD_CODER_MOVE_POS(pCoder, sizeof(val));
return 0;
static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); }
static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); }
static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) {
while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
// 16
static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) { TD_ENCODE_MACRO(pCoder, val, uint16_t, 16); }
static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) { TD_ENCODE_MACRO(pCoder, val, int16_t, 16); }
// 32
static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) { TD_ENCODE_MACRO(pCoder, val, uint32_t, 32); }
static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) { TD_ENCODE_MACRO(pCoder, val, int32_t, 32); }
// 64
static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) { TD_ENCODE_MACRO(pCoder, val, uint64_t, 64); }
static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) { TD_ENCODE_MACRO(pCoder, val, int64_t, 64); }
// 16v
static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) {
return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val));
}
// 32v
static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) {
while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) {
return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val));
}
// 64v
static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) {
while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) {
return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val));
}
@ -265,14 +206,16 @@ static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) {
}
static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) {
if (tEncodeU32v(pCoder, len) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeU32v(pCoder, len));
if (len) {
if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) return -1;
memcpy(TD_CODER_CURRENT(pCoder), val, len);
if (pCoder->pos + len > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
memcpy(pCoder->data + pCoder->pos, val, len);
}
TD_CODER_MOVE_POS(pCoder, len);
pCoder->pos += len;
}
return 0;
}
@ -286,70 +229,137 @@ static FORCE_INLINE int32_t tEncodeCStr(SEncoder* pCoder, const char* val) {
}
/* ------------------------ FOR DECODER ------------------------ */
// 8
static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1;
tGet(uint8_t, TD_CODER_CURRENT(pCoder), *val);
TD_CODER_MOVE_POS(pCoder, sizeof(*val));
static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size) {
if (pCoder->pos + size > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} else if (val) {
memcpy(val, pCoder->data + pCoder->pos, size);
}
pCoder->pos += size;
return 0;
}
static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1;
tGet(int8_t, TD_CODER_CURRENT(pCoder), *val);
TD_CODER_MOVE_POS(pCoder, sizeof(*val));
return 0;
static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { return tDecodeFixed(pCoder, val, sizeof(*val)); }
// 16
static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) { TD_DECODE_MACRO(pCoder, val, uint16_t, 16); }
static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) { TD_DECODE_MACRO(pCoder, val, int16_t, 16); }
static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 32
static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) { TD_DECODE_MACRO(pCoder, val, uint32_t, 32); }
static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) { TD_DECODE_MACRO(pCoder, val, int32_t, 32); }
static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 64
static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) { TD_DECODE_MACRO(pCoder, val, uint64_t, 64); }
static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) { TD_DECODE_MACRO(pCoder, val, int64_t, 64); }
static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 16v
static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint16_t);
uint8_t byte;
uint16_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint16_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint16_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
}
static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) {
uint16_t tval;
if (tDecodeU16v(pCoder, &tval) < 0) {
return -1;
TAOS_CHECK_RETURN(tDecodeU16v(pCoder, &tval));
if (val) {
*val = ZIGZAGD(int16_t, tval);
}
if (val) *val = ZIGZAGD(int16_t, tval);
return 0;
}
// 32v
static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint32_t);
uint8_t byte;
uint32_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint32_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint32_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
}
static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) {
uint32_t tval;
if (tDecodeU32v(pCoder, &tval) < 0) {
return -1;
TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &tval));
if (val) {
*val = ZIGZAGD(int32_t, tval);
}
if (val) *val = ZIGZAGD(int32_t, tval);
return 0;
}
// 64v
static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint64_t);
uint8_t byte;
uint64_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint64_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint64_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
}
static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) {
uint64_t tval;
if (tDecodeU64v(pCoder, &tval) < 0) {
return -1;
TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &tval));
if (val) {
*val = ZIGZAGD(int64_t, tval);
}
if (val) *val = ZIGZAGD(int64_t, tval);
return 0;
}
@ -359,11 +369,11 @@ static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) {
float f;
} v;
if (tDecodeU32(pCoder, &(v.ui)) < 0) {
return -1;
}
TAOS_CHECK_RETURN(tDecodeU32(pCoder, &(v.ui)));
*val = v.f;
if (val) {
*val = v.f;
}
return 0;
}
@ -373,30 +383,36 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) {
double d;
} v;
if (tDecodeU64(pCoder, &(v.ui)) < 0) {
return -1;
}
TAOS_CHECK_RETURN(tDecodeU64(pCoder, &(v.ui)));
*val = v.d;
if (val) {
*val = v.d;
}
return 0;
}
static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) {
uint32_t length = 0;
if (tDecodeU32v(pCoder, &length) < 0) return -1;
if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1;
if (val) {
*val = (uint8_t*)TD_CODER_CURRENT(pCoder);
TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length));
if (len) {
*len = length;
}
TD_CODER_MOVE_POS(pCoder, length);
if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
if (val) {
*val = pCoder->data + pCoder->pos;
}
pCoder->pos += length;
return 0;
}
static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) {
if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeBinary(pCoder, (uint8_t**)val, len));
(*len) -= 1;
return 0;
}
@ -409,7 +425,7 @@ static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) {
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
char* pStr;
uint32_t len;
if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len));
memcpy(val, pStr, len + 1);
return 0;
@ -417,16 +433,22 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) {
uint64_t length = 0;
if (tDecodeU64v(pCoder, &length) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &length));
if (length) {
if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1;
*val = taosMemoryMalloc(length);
if (*val == NULL) return -1;
memcpy(*val, TD_CODER_CURRENT(pCoder), length);
if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
TD_CODER_MOVE_POS(pCoder, length);
*val = taosMemoryMalloc(length);
if (*val == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*val, pCoder->data + pCoder->pos, length);
pCoder->pos += length;
} else {
*val = NULL;
}
@ -435,16 +457,20 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin
static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) {
uint32_t length = 0;
if (tDecodeU32v(pCoder, &length) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length));
if (length) {
if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1;
if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
*val = taosMemoryMalloc(length);
if (*val == NULL) return -1;
memcpy(*val, TD_CODER_CURRENT(pCoder), length);
if (*val == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*val, pCoder->data + pCoder->pos, length);
TD_CODER_MOVE_POS(pCoder, length);
pCoder->pos += length;
} else {
*val = NULL;
}
@ -452,7 +478,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u
}
static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) {
if (tDecodeBinaryAlloc(pCoder, (void**)val, len) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeBinaryAlloc(pCoder, (void**)val, len));
(*len) -= 1;
return 0;
}

24
include/util/tscalablebf.h
View File

@ -23,22 +23,22 @@ extern "C" {
#endif
typedef struct SScalableBf {
SArray *bfArray; // array of bloom filters
uint32_t growth;
uint64_t numBits;
uint32_t maxBloomFilters;
int8_t status;
SArray* bfArray; // array of bloom filters
uint32_t growth;
uint64_t numBits;
uint32_t maxBloomFilters;
int8_t status;
_hash_fn_t hashFn1;
_hash_fn_t hashFn2;
} SScalableBf;
SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate);
int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf *pSBf);
int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder);
SScalableBf *tScalableBfDecode(SDecoder *pDecoder);
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf);
int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf* pSBf);
int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder);
int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf);
#ifdef __cplusplus
}

1
include/util/tutil.h
View File

@ -21,6 +21,7 @@
#include "tdef.h"
#include "thash.h"
#include "tmd5.h"
#include "tutil.h"
#ifdef __cplusplus
extern "C" {

18
source/client/inc/clientSml.h
View File

@ -82,7 +82,7 @@ extern "C" {
#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH))
#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
#define MOVE_FORWARD_ONE(sql, len) ((void)memmove((void *)((sql)-1), (sql), len))
#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \
@ -211,19 +211,19 @@ extern int64_t smlFactorS[];
typedef int32_t (*_equal_fn_sml)(const void *, const void *);
SSmlHandle *smlBuildSmlInfo(TAOS *taos);
int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle);
void smlDestroyInfo(SSmlHandle *info);
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen);
SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat);
int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo);
int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta** sMeta);
int32_t smlSetCTableName(SSmlTableInfo *oneTable);
void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo);
STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen);
int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo);
int32_t smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen, STableMeta **pTableMeta);
int32_t is_same_child_table_telnet(const void *a, const void *b);
int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len);
int32_t smlClearForRerun(SSmlHandle *info);
@ -236,7 +236,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseJSON(SSmlHandle *info, char *payload);
SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement);
int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta);
bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);

23
source/client/src/clientRawBlockWrite.c
View File

@ -1572,9 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
}
// uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid);
pQuery = smlInitHandle();
if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = smlInitHandle(&pQuery);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
@ -1656,9 +1655,9 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
pQuery = smlInitHandle();
if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = smlInitHandle(&pQuery);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
@ -1755,12 +1754,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle();
if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = smlInitHandle(&pQuery);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("init sml handle failed");
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter);
@ -1903,12 +1902,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle();
if (pQuery == NULL) {
code = smlInitHandle(&pQuery);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("init sml handle failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum);

637
source/client/src/clientSml.c
View File

File diff suppressed because it is too large Load Diff

118
source/client/src/clientSmlJson.c
View File

@ -29,29 +29,29 @@
(start)++; \
}
static char *smlJsonGetObj(char *payload) {
static int32_t smlJsonGetObj(char **payload) {
int leftBracketCnt = 0;
bool isInQuote = false;
while (*payload) {
if (*payload == '"' && *(payload - 1) != '\\') {
while (**payload) {
if (**payload == '"' && *((*payload) - 1) != '\\') {
isInQuote = !isInQuote;
} else if (!isInQuote && unlikely(*payload == '{')) {
} else if (!isInQuote && unlikely(**payload == '{')) {
leftBracketCnt++;
payload++;
(*payload)++;
continue;
} else if (!isInQuote && unlikely(*payload == '}')) {
} else if (!isInQuote && unlikely(**payload == '}')) {
leftBracketCnt--;
payload++;
(*payload)++;
if (leftBracketCnt == 0) {
return payload;
return 0;
} else if (leftBracketCnt < 0) {
return NULL;
return -1;
}
continue;
}
payload++;
(*payload)++;
}
return NULL;
return -1;
}
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
@ -99,8 +99,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
offset[index++] = *start - sTmp;
element->timestamp = (*start);
if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start);
*start = tmp;
}
@ -127,8 +128,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
offset[index++] = *start - sTmp;
element->cols = (*start);
if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start);
*start = tmp;
}
@ -153,8 +155,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
JUMP_JSON_SPACE((*start))
offset[index++] = *start - sTmp;
element->tags = (*start);
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start);
*start = tmp;
}
@ -209,8 +212,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
(*start) += offset[index++];
element->timestamp = *start;
if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start);
*start = tmp;
}
@ -227,8 +231,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
(*start) += offset[index++];
element->cols = *start;
if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start);
*start = tmp;
}
@ -244,8 +249,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
} else if ((*start)[1] == 't' && (*start)[2] == 'a') {
(*start) += offset[index++];
element->tags = (*start);
char *tmp = smlJsonGetObj((*start));
if (tmp) {
char *tmp = *start;
int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start);
*start = tmp;
}
@ -261,7 +267,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
uError("elements != %d", OTD_JSON_FIELDS_NUM);
return TSDB_CODE_TSC_INVALID_JSON;
}
return 0;
return TSDB_CODE_SUCCESS;
}
static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) {
@ -466,7 +472,11 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
break;
}
case cJSON_String: {
smlConvertJSONString(kv, "binary", root);
int32_t ret = smlConvertJSONString(kv, "binary", root);
if (ret != TSDB_CODE_SUCCESS) {
uError("OTD:Failed to parse binary value from JSON Obj");
return ret;
}
break;
}
case cJSON_Object: {
@ -492,20 +502,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
int32_t tagNum = cJSON_GetArraySize(tags);
if (unlikely(tagNum == 0)) {
uError("SML:Tag should not be empty");
terrno = TSDB_CODE_TSC_INVALID_JSON;
return -1;
return TSDB_CODE_TSC_INVALID_JSON;
}
for (int32_t i = 0; i < tagNum; ++i) {
cJSON *tag = cJSON_GetArrayItem(tags, i);
if (unlikely(tag == NULL)) {
terrno = TSDB_CODE_TSC_INVALID_JSON;
return -1;
return TSDB_CODE_TSC_INVALID_JSON;
}
size_t keyLen = strlen(tag->string);
if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
uError("OTD:Tag key length is 0 or too large than 64");
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
// add kv to SSmlKv
@ -516,19 +523,19 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
// value
int32_t ret = smlParseValueFromJSON(tag, &kv);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
terrno = ret;
return -1;
return ret;
}
if (taosArrayPush(preLineKV, &kv) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(preLineKV, &kv);
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS;
return -1;
return TSDB_CODE_TSC_INVALID_JSON;
}
cnt++;
}
return 0;
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
@ -536,12 +543,18 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
if(info->dataFormat){
ret = smlProcessSuperTable(info, elements);
if(ret != 0){
return terrno;
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
}
ret = smlProcessTagJson(info, tags);
if(ret != 0){
return terrno;
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
ret = smlJoinMeasureTag(elements);
if(ret != 0){
@ -703,6 +716,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
// Parse tags
bool needFree = info->dataFormat;
elements->tags = cJSON_PrintUnformatted(tagsJson);
if (elements->tags == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
elements->tagsLen = strlen(elements->tags);
if (is_same_child_table_telnet(elements, &info->preLine) != 0) {
ret = smlParseTagsFromJSON(info, tagsJson, elements);
@ -729,7 +745,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
// notice!!! put ts back to tag to ensure get meta->precision
int64_t ts = smlParseTSFromJSON(info, tsJson);
if (unlikely(ts < 0)) {
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id);
char* tmp = cJSON_PrintUnformatted(tsJson);
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts);
return TSDB_CODE_INVALID_TIMESTAMP;
}
SSmlKv kvTs = {0};
@ -823,7 +840,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
}
if (ret != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_VALUE;
return ret;
}
if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS;
@ -844,18 +861,23 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
cJSON *valueJson = cJSON_Parse(elements->cols);
if (unlikely(valueJson == NULL)) {
uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_JSON;
}
taosArrayPush(info->tagJsonArray, &valueJson);
if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){
cJSON_Delete(valueJson);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_OUT_OF_MEMORY;
}
ret = smlParseValueFromJSONObj(valueJson, &kv);
if (ret != TSDB_CODE_SUCCESS) {
uError("SML:Failed to parse value from JSON Obj:%s", elements->cols);
uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_VALUE;
}
elements->cols[elements->colsLen] = tmp;
} else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) {
uError("SML:cols invalidate:%s", elements->cols);
uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols);
return TSDB_CODE_TSC_INVALID_VALUE;
}
@ -870,7 +892,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
return TSDB_CODE_TSC_INVALID_JSON;
}
taosArrayPush(info->tagJsonArray, &tagsJson);
if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){
cJSON_Delete(tagsJson);
uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id);
return TSDB_CODE_OUT_OF_MEMORY;
}
ret = smlParseTagsFromJSON(info, tagsJson, elements);
if (unlikely(ret)) {
uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id);
@ -937,7 +963,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) {
return ret;
}
info->lines = (SSmlLineInfo *)tmp;
memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo));
(void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo));
}
ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt);
if ((info->lines + cnt)->measure == NULL) break;

68
source/client/src/clientSmlLine.c
View File

@ -103,7 +103,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
return code;
}
char* tmp = taosMemoryCalloc(pVal->length, 1);
memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN);
code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length);
taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) {
@ -149,7 +152,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
if(data == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length);
(void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length);
pVal->value = data;
}
@ -212,8 +215,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
while (*sql < sqlEnd) {
if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(IS_EQUAL(*sql,escapeChar))) {
keyLen = *sql - key;
@ -230,8 +232,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
// parse value
@ -245,8 +246,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
break;
} else if (unlikely(IS_EQUAL(*sql,escapeChar))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
@ -261,24 +261,28 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
if (unlikely(valueLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
return -1;
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
if (keyEscaped) {
char *tmp = (char *)taosMemoryMalloc(keyLen);
memcpy(tmp, key, keyLen);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, key, keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen);
key = tmp;
}
if (valueEscaped) {
char *tmp = (char *)taosMemoryMalloc(valueLen);
memcpy(tmp, value, valueLen);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, value, valueLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen);
value = tmp;
}
@ -289,11 +293,12 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
.length = valueLen,
.keyEscaped = keyEscaped,
.valueEscaped = valueEscaped};
taosArrayPush(preLineKV, &kv);
if(taosArrayPush(preLineKV, &kv) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS;
return -1;
return TSDB_CODE_TSC_INVALID_JSON;
}
cnt++;
@ -302,7 +307,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
}
(*sql)++;
}
return 0;
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
@ -315,13 +320,19 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if(info->dataFormat){
ret = smlProcessSuperTable(info, elements);
if(ret != 0){
return terrno;
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
}
ret = smlProcessTagLine(info, sql, sqlEnd);
if(ret != 0){
return terrno;
if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
return smlProcessChildTable(info, elements);
@ -410,7 +421,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if (keyEscaped) {
char *tmp = (char *)taosMemoryMalloc(kv.keyLen);
memcpy(tmp, key, kv.keyLen);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, key, kv.keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen);
kv.key = tmp;
kv.keyEscaped = keyEscaped;
@ -418,7 +432,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if (valueEscaped) {
char *tmp = (char *)taosMemoryMalloc(kv.length);
memcpy(tmp, kv.value, kv.length);
if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, kv.value, kv.length);
PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY);
if(kv.type == TSDB_DATA_TYPE_VARBINARY){
@ -437,8 +454,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
} else {
if (currElement->colArray == NULL) {
currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
if (currElement->colArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(currElement->colArray, &kv); // reserve for timestamp
}
cnt++;

34
source/client/src/clientSmlTelnet.c
View File

@ -84,8 +84,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
while (sql < sqlEnd) {
if (unlikely(*sql == SPACE)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
if (unlikely(*sql == EQUAL)) {
keyLen = sql - key;
@ -97,8 +96,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
}
// parse value
@ -111,8 +109,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
}
if (unlikely(*sql == EQUAL)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
terrno = TSDB_CODE_SML_INVALID_DATA;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
sql++;
}
@ -120,13 +117,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
if (unlikely(valueLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
terrno = TSDB_CODE_TSC_INVALID_VALUE;
return -1;
return TSDB_CODE_TSC_INVALID_VALUE;
}
if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
return -1;
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
SSmlKv kv = {.key = key,
@ -136,14 +131,15 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
.length = valueLen,
.keyEscaped = false,
.valueEscaped = false};
taosArrayPush(preLineKV, &kv);
if (taosArrayPush(preLineKV, &kv) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS;
return -1;
return TSDB_CODE_SML_INVALID_DATA;
}
cnt++;
}
return 0;
return TSDB_CODE_SUCCESS;
}
static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
@ -156,13 +152,19 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
if(info->dataFormat){
ret = smlProcessSuperTable(info, elements);
if(ret != 0){
return terrno;
if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
}
ret = smlProcessTagTelnet(info, data, sqlEnd);
if(ret != 0){
return terrno;
if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
}
ret = smlJoinMeasureTag(elements);

114
source/client/test/smlTest.cpp
View File

@ -40,13 +40,16 @@ TEST(testCase, smlParseInfluxString_Test) {
msgBuf.len = 256;
SSmlLineInfo elements = {0};
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, 0);
info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false;
// case 1
char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3";
char *sql = (char *)taosMemoryCalloc(256, 1);
memcpy(sql, tmp, strlen(tmp) + 1);
ASSERT_NE(sql, nullptr);
(void)memcpy(sql, tmp, strlen(tmp) + 1);
int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql);
@ -62,22 +65,22 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 2 false
tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 4 tag is null
tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql);
@ -92,13 +95,13 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 5 tag is null
tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql + 1);
@ -112,36 +115,36 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 6
tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0);
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
smlClearForRerun(info);
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
ASSERT_EQ(smlClearForRerun(info), 0);
// case 7
tmp = " st , ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
// case 8 false
tmp = ", st , ";
memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo));
(void)memcpy(sql, tmp, strlen(tmp) + 1);
(void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray);
elements.colArray = NULL;
(void)taosArrayDestroy(elements.colArray);
elements.colArray = nullptr;
taosMemoryFree(sql);
smlDestroyInfo(info);
@ -228,7 +231,9 @@ TEST(testCase, smlParseCols_Error_Test) {
"escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000"
};
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false;
for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) {
@ -238,13 +243,14 @@ TEST(testCase, smlParseCols_Error_Test) {
msgBuf.len = 256;
int32_t len = strlen(data[i]);
char *sql = (char *)taosMemoryCalloc(256, 1);
memcpy(sql, data[i], len + 1);
ASSERT_NE(sql, nullptr);
(void)memcpy(sql, data[i], len + 1);
SSmlLineInfo elements = {0};
int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements);
// printf("i:%d\n", i);
ASSERT_NE(ret, TSDB_CODE_SUCCESS);
taosMemoryFree(sql);
taosArrayDestroy(elements.colArray);
(void)taosArrayDestroy(elements.colArray);
}
smlDestroyInfo(info);
}
@ -254,7 +260,9 @@ TEST(testCase, smlParseCols_Test) {
SSmlMsgBuf msgBuf;
msgBuf.buf = msg;
msgBuf.len = 256;
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false;
SSmlLineInfo elements = {0};
@ -267,7 +275,8 @@ TEST(testCase, smlParseCols_Test) {
"cboolf=f,cnch_=l\"iuwq\" 1626006833639000000";
int32_t len = strlen(data);
char *sql = (char *)taosMemoryCalloc(1024, 1);
memcpy(sql, data, len + 1);
ASSERT_NE(sql, nullptr);
(void)memcpy(sql, data, len + 1);
int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements);
ASSERT_EQ(ret, TSDB_CODE_SUCCESS);
int32_t size = taosArrayGetSize(elements.colArray);
@ -275,6 +284,7 @@ TEST(testCase, smlParseCols_Test) {
// binary
SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0);
ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY);
@ -285,6 +295,7 @@ TEST(testCase, smlParseCols_Test) {
// nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 2);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
@ -293,6 +304,7 @@ TEST(testCase, smlParseCols_Test) {
// bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 3);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0);
ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -301,6 +313,7 @@ TEST(testCase, smlParseCols_Test) {
// double
kv = (SSmlKv *)taosArrayGet(elements.colArray, 4);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
@ -310,6 +323,7 @@ TEST(testCase, smlParseCols_Test) {
// float
kv = (SSmlKv *)taosArrayGet(elements.colArray, 5);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0);
ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
@ -319,6 +333,7 @@ TEST(testCase, smlParseCols_Test) {
// float
kv = (SSmlKv *)taosArrayGet(elements.colArray, 6);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT);
@ -328,6 +343,7 @@ TEST(testCase, smlParseCols_Test) {
// tiny int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 7);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0);
ASSERT_EQ(kv->keyLen, 3);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT);
@ -336,6 +352,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned tiny int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 8);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0);
ASSERT_EQ(kv->keyLen, 3);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT);
@ -344,6 +361,7 @@ TEST(testCase, smlParseCols_Test) {
// small int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 9);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT);
@ -352,6 +370,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned smallint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 10);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT);
@ -360,6 +379,7 @@ TEST(testCase, smlParseCols_Test) {
// int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 11);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT);
@ -368,6 +388,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 12);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT);
@ -376,6 +397,7 @@ TEST(testCase, smlParseCols_Test) {
// bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 13);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT);
@ -384,6 +406,7 @@ TEST(testCase, smlParseCols_Test) {
// bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 14);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0);
ASSERT_EQ(kv->keyLen, 2);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT);
@ -392,6 +415,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 15);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT);
@ -400,6 +424,7 @@ TEST(testCase, smlParseCols_Test) {
// bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 16);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0);
ASSERT_EQ(kv->keyLen, 9);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -408,6 +433,7 @@ TEST(testCase, smlParseCols_Test) {
// bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 17);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0);
ASSERT_EQ(kv->keyLen, 6);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -416,6 +442,7 @@ TEST(testCase, smlParseCols_Test) {
// bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 18);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0);
ASSERT_EQ(kv->keyLen, 6);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -424,13 +451,14 @@ TEST(testCase, smlParseCols_Test) {
// nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 19);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0);
ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->length, 4);
ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0);
taosArrayDestroy(elements.colArray);
(void)taosArrayDestroy(elements.colArray);
taosMemoryFree(sql);
smlDestroyInfo(info);
}
@ -471,7 +499,9 @@ TEST(testCase, smlParseNumber_Test) {
}
TEST(testCase, smlParseTelnetLine_error_Test) {
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->dataFormat = false;
info->protocol = TSDB_SML_TELNET_PROTOCOL;
ASSERT_NE(info, nullptr);
@ -509,7 +539,9 @@ TEST(testCase, smlParseTelnetLine_error_Test) {
}
TEST(testCase, smlParseTelnetLine_Test) {
SSmlHandle *info = smlBuildSmlInfo(NULL);
SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->dataFormat = false;
info->protocol = TSDB_SML_TELNET_PROTOCOL;
ASSERT_NE(info, nullptr);
@ -528,7 +560,9 @@ TEST(testCase, smlParseTelnetLine_Test) {
}
//TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) {
// SSmlHandle *info = smlBuildSmlInfo(NULL);
// SSmlHandle *info = nullptr;
// int32_t code = smlBuildSmlInfo(nullptr, &info);
// ASSERT_EQ(code, TSDB_CODE_SUCCESS);
// info->protocol = TSDB_SML_JSON_PROTOCOL;
// ASSERT_NE(info, nullptr);
//
@ -568,7 +602,7 @@ TEST(testCase, smlParseNumber_performance_Test) {
for (int j = 0; j < 10000000; ++j) {
kv.value = str[i];
kv.length = strlen(str[i]);
smlParseNumber(&kv, &msgBuf);
(void)smlParseNumber(&kv, &msgBuf); //ignore result
}
printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1);
printf("\n");
@ -576,7 +610,7 @@ TEST(testCase, smlParseNumber_performance_Test) {
for (int j = 0; j < 10000000; ++j) {
kv.value = str[i];
kv.length = strlen(str[i]);
smlParseNumberOld(&kv, &msgBuf);
(void)smlParseNumberOld(&kv, &msgBuf); //ignore result
}
printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2);
printf("\n\n");

183
source/common/src/cos.c
View File

@ -3,6 +3,7 @@
#include "cos.h"
#include "cos_cp.h"
#include "tdef.h"
#include "tutil.h"
extern char tsS3Endpoint[];
extern char tsS3AccessKeyId[];
@ -40,7 +41,7 @@ int32_t s3Begin() {
if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) {
uError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
protocolG = !tsS3Https;
@ -48,12 +49,12 @@ int32_t s3Begin() {
uriStyleG = S3UriStyleVirtualHost;
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
void s3End() { S3_deinitialize(); }
int32_t s3Init() { return 0; /*s3Begin();*/ }
int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); /*s3Begin();*/ }
void s3CleanUp() { /*s3End();*/
}
@ -61,7 +62,7 @@ void s3CleanUp() { /*s3End();*/
static int32_t s3ListBucket(char const *bucketname);
int32_t s3CheckCfg() {
int32_t code = 0;
int32_t code = 0, lino = 0;
if (!tsS3Enabled) {
fprintf(stderr, "s3 not configured.\n");
@ -71,7 +72,7 @@ int32_t s3CheckCfg() {
code = s3Begin();
if (code != 0) {
fprintf(stderr, "failed to initialize s3.\n");
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
// test put
@ -91,20 +92,17 @@ int32_t s3CheckCfg() {
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (!fp) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to open test file: %s.\n", path);
// uError("ERROR: %s Failed to open %s", __func__, path);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to write test file: %s.\n", path);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to fsync test file: %s.\n", path);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
taosCloseFile(&fp);
@ -112,7 +110,7 @@ int32_t s3CheckCfg() {
code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16);
if (code != 0) {
fprintf(stderr, "put object %s : failed.\n", objectname[0]);
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
fprintf(stderr, "put object %s: success.\n\n", objectname[0]);
@ -121,7 +119,7 @@ int32_t s3CheckCfg() {
code = s3ListBucket(tsS3BucketName);
if (code != 0) {
fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName);
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName);
@ -134,7 +132,7 @@ int32_t s3CheckCfg() {
code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
if (code != 0) {
fprintf(stderr, "get object %s : failed.\n", objectname[0]);
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
char buf[7] = {0};
memcpy(buf, pBlock, c_len);
@ -147,14 +145,14 @@ int32_t s3CheckCfg() {
code = s3DeleteObjects(objectname, 1);
if (code != 0) {
fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
goto _exit;
TAOS_CHECK_GOTO(code, &lino, _exit);
}
fprintf(stderr, "delete object %s: success.\n\n", objectname[0]);
s3End();
_exit:
return code;
TAOS_RETURN(code);
}
static int should_retry() {
@ -169,7 +167,7 @@ static int should_retry() {
}
*/
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status,
@ -202,7 +200,6 @@ static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *prope
}
static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) {
//(void)callbackData;
TS3SizeCBD *cbd = callbackData;
if (properties->contentLength > 0) {
cbd->content_length = properties->contentLength;
@ -249,7 +246,7 @@ static int32_t s3ListBucket(char const *bucketname) {
SArray *objectArray = getListByPrefix("s3");
if (objectArray == NULL) {
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
const char **object_name = TARRAY_DATA(objectArray);
@ -262,7 +259,7 @@ static int32_t s3ListBucket(char const *bucketname) {
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return code;
TAOS_RETURN(code);
}
typedef struct growbuffer {
@ -607,13 +604,13 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c
code = TAOS_SYSTEM_ERROR(EIO);
}
return code;
TAOS_RETURN(code);
}
static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name,
int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) {
int32_t code = 0;
int32_t code = 0, lino = 0;
uint64_t totalContentLength = contentLength;
uint64_t todoContentLength = contentLength;
UploadManager manager = {0};
@ -640,6 +637,9 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
if (!manager.etags) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
manager.next_etags_pos = 0;
do {
S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
@ -647,8 +647,7 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
upload:
@ -672,8 +671,7 @@ upload:
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
contentLength -= chunk_size;
todoContentLength -= chunk_size;
@ -686,8 +684,7 @@ upload:
int n;
for (i = 0; i < totalSeq; i++) {
if (!manager.etags[i]) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
n = snprintf(buf, sizeof(buf),
"<Part><PartNumber>%d</PartNumber>"
@ -704,11 +701,13 @@ upload:
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
clean:
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (manager.upload_id) {
taosMemoryFree(manager.upload_id);
}
@ -718,13 +717,13 @@ clean:
growbuffer_destroy(manager.gb);
taosMemoryFree(manager.etags);
return code;
TAOS_RETURN(code);
}
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) {
int32_t code = 0;
int32_t code = 0, lino = 0;
uint64_t totalContentLength = contentLength;
// uint64_t todoContentLength = contentLength;
@ -744,13 +743,16 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
SCheckpoint cp = {0};
cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
if (!cp.parts) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (taosCheckExistFile(file_cp_path)) {
if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) {
manager.upload_id = strdup(cp.upload_id);
need_init_upload = false;
} else {
cos_cp_remove(file_cp_path);
TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit);
}
}
@ -763,16 +765,14 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size);
}
if (cos_cp_open(file_cp_path, &cp)) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
int part_num = 0;
@ -791,6 +791,10 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
if (!manager.etags) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
manager.next_etags_pos = 0;
upload:
@ -802,8 +806,7 @@ upload:
if (i > 0 && cp.parts[i - 1].completed) {
if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
}
@ -827,25 +830,23 @@ upload:
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
//(void)cos_cp_dump(&cp);
goto clean;
}
if (!manager.etags[seq - 1]) {
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0);
(void)cos_cp_dump(&cp);
TAOS_CHECK_GOTO(cos_cp_dump(&cp), &lino, _exit);
contentLength -= chunk_size;
// todoContentLength -= chunk_size;
}
cos_cp_close(cp.thefile);
TAOS_CHECK_GOTO(cos_cp_close(cp.thefile), &lino, _exit);
cp.thefile = 0;
int size = 0;
@ -869,20 +870,23 @@ upload:
} while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO);
goto clean;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
}
cos_cp_remove(file_cp_path);
TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit);
clean:
_exit:
/*
if (parts) {
taosMemoryFree(parts);
}
*/
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (cp.thefile) {
cos_cp_close(cp.thefile);
(void)cos_cp_close(cp.thefile);
}
if (cp.parts) {
taosMemoryFree(cp.parts);
@ -899,7 +903,7 @@ clean:
taosMemoryFree(manager.etags);
growbuffer_destroy(manager.gb);
return code;
TAOS_RETURN(code);
}
int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) {
@ -917,15 +921,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
@ -956,7 +958,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
growbuffer_destroy(data.gb);
}
return code;
TAOS_RETURN(code);
}
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
@ -974,22 +976,19 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int
put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
contentLength = size;
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) {
taosCloseFile(&data.infileFD);
code = TAOS_SYSTEM_ERROR(errno);
return code;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
@ -1016,7 +1015,7 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int
growbuffer_destroy(data.gb);
}
return code;
TAOS_RETURN(code);
}
typedef struct list_bucket_callback_data {
@ -1130,11 +1129,11 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
code = -1;
code = TSDB_CODE_FAILED;
}
}
return code;
TAOS_RETURN(code);
}
void s3DeleteObjectsByPrefix(const char *prefix) {
@ -1187,17 +1186,19 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
if (cbd.status != S3StatusOK) {
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
}
if (check && cbd.buf_pos != size) {
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
}
*ppBlock = cbd.buf;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) {
@ -1218,9 +1219,8 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) {
TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("[s3] open file error, errno:%d, fileName:%s", terrno, fileName);
return -1;
uError("[s3] open file error, errno:%d, fileName:%s", TAOS_SYSTEM_ERROR(errno), fileName);
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
TS3GetData cbd = {0};
@ -1232,16 +1232,17 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) {
if (cbd.status != S3StatusOK) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
taosCloseFile(&pFile);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
}
taosCloseFile(&pFile);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
SArray *objectArray = getListByPrefix(prefix);
if (objectArray == NULL) return -1;
if (objectArray == NULL) TAOS_RETURN(TSDB_CODE_FAILED);
for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) {
char *object = taosArrayGetP(objectArray, i);
@ -1255,7 +1256,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
}
if (s3GetObjectToFile(object, fileName) != 0) {
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
}
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
@ -1279,7 +1280,7 @@ long s3Size(const char *object_name) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
return -1;
TAOS_RETURN(TSDB_CODE_FAILED);
}
size = cbd.content_length;
@ -1306,7 +1307,7 @@ int32_t s3Init() {
// set log output, default stderr
cos_log_set_output(NULL);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
void s3CleanUp() { cos_http_io_deinitialize(); }
@ -1363,10 +1364,10 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
cos_pool_destroy(p);
if (s->code != 200) {
return code = s->code;
TAOS_RETURN(s->code);
}
return code;
TAOS_RETURN(code);
}
int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) {
@ -1405,10 +1406,10 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_
cos_pool_destroy(p);
if (s->code != 200) {
return code = s->code;
TAOS_RETURN(s->code);
}
return code;
TAOS_RETURN(code);
}
void s3DeleteObjectsByPrefix(const char *prefix_str) {
@ -1465,7 +1466,7 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) {
cos_warn_log("delete objects failed\n");
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
bool s3Exists(const char *object_name) {
@ -1547,7 +1548,7 @@ bool s3Get(const char *object_name, const char *path) {
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) {
(void)check;
int32_t code = 0;
int32_t code = 0, lino = 0;
cos_pool_t *p = NULL;
int is_cname = 0;
cos_status_t *s = NULL;
@ -1588,10 +1589,9 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
s = cos_get_object_to_buffer(options, &bucket, &object, headers, NULL, &download_buffer, &resp_headers);
log_status(s);
if (!cos_status_is_ok(s)) {
vError("s3: %d(%s)", s->code, s->error_msg);
vError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
code = TAOS_SYSTEM_ERROR(EIO);
return code;
uError("s3: %d(%s)", s->code, s->error_msg);
uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
}
// print_headers(resp_headers);
@ -1601,6 +1601,10 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { len += cos_buf_size(content); }
// char *buf = cos_pcalloc(p, (apr_size_t)(len + 1));
char *buf = taosMemoryCalloc(1, (apr_size_t)(len));
if (!buf) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
// buf[len] = '\0';
cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) {
size = cos_buf_size(content);
@ -1609,12 +1613,13 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
}
// cos_warn_log("Download data=%s", buf);
_exit:
//销毁内存池
cos_pool_destroy(p);
*ppBlock = buf;
return code;
TAOS_RETURN(code);
}
typedef struct {

172
source/common/src/cos_cp.c
View File

@ -9,42 +9,35 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH);
if (!fd) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open %s", __func__, cp_path);
return code;
uError("%s Failed to open %s", __func__, cp_path);
TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno));
}
checkpoint->thefile = fd;
return code;
TAOS_RETURN(code);
}
void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); }
void cos_cp_remove(char const* filepath) { (void)taosRemoveFile(filepath); }
int32_t cos_cp_close(TdFilePtr fd) { return taosCloseFile(&fd); }
int32_t cos_cp_remove(char const* filepath) { return taosRemoveFile(filepath); }
static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
int32_t code = 0;
int32_t code = 0, lino = 0;
cJSON const* item2 = NULL;
cJSON* json = cJSON_Parse(cp_body);
if (NULL == json) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
}
cJSON const* item = cJSON_GetObjectItem(json, "ver");
if (!cJSON_IsNumber(item) || item->valuedouble != 1) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble);
goto _exit;
TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
}
item = cJSON_GetObjectItem(json, "type");
if (!cJSON_IsNumber(item)) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
}
cp->cp_type = item->valuedouble;
@ -155,40 +148,36 @@ static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
}
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (json) cJSON_Delete(json);
if (cp_body) taosMemoryFree(cp_body);
return code;
TAOS_RETURN(code);
}
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
int32_t code = 0;
int32_t code = 0, lino = 0;
char* cp_body = NULL;
TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ);
if (!fd) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open %s", __func__, filepath);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
int64_t size = -1;
code = taosStatFile(filepath, &size, NULL, NULL);
if (code) {
uError("ERROR: %s Failed to stat %s", __func__, filepath);
goto _exit;
}
TAOS_CHECK_GOTO(taosStatFile(filepath, &size, NULL, NULL), &lino, _exit);
char* cp_body = taosMemoryMalloc(size + 1);
cp_body = taosMemoryMalloc(size + 1);
if (!cp_body) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
int64_t n = taosReadFile(fd, cp_body, size);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to read %s", __func__, filepath);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
} else if (n != size) {
code = TSDB_CODE_FILE_CORRUPTED;
uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size);
goto _exit;
TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
}
taosCloseFile(&fd);
cp_body[size] = '\0';
@ -196,184 +185,161 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
return cos_cp_parse_body(cp_body, checkpoint);
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (fd) {
taosCloseFile(&fd);
}
if (cp_body) {
taosMemoryFree(cp_body);
}
return code;
TAOS_RETURN(code);
}
static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
int32_t code = 0;
int32_t code = 0, lino = 0;
char* data = cJSON_PrintUnformatted(json);
if (NULL == data) {
return TSDB_CODE_OUT_OF_MEMORY;
if (!data) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
TdFilePtr fp = checkpoint->thefile;
if (taosFtruncateFile(fp, 0) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosLSeekFile(fp, 0, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosWriteFile(fp, data, strlen(data)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosMemoryFree(data);
return code;
TAOS_RETURN(code);
}
int32_t cos_cp_dump(SCheckpoint* cp) {
int32_t code = 0;
int32_t lino = 0;
int32_t code = 0, lino = 0;
cJSON* ojson = NULL;
cJSON* json = cJSON_CreateObject();
if (!json) return TSDB_CODE_OUT_OF_MEMORY;
if (!json) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (COS_CP_TYPE_UPLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "file");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
} else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "object");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
}
ojson = cJSON_AddObjectToObject(json, "cpparts");
if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts");
if (!ajson) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
for (int i = 0; i < cp->part_num; ++i) {
cJSON* item = cJSON_CreateObject();
if (!item) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
cJSON_AddItemToArray(ajson, item);
if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
}
code = cos_cp_save_json(json, cp);
TSDB_CHECK_CODE(code, lino, _exit);
TAOS_CHECK_GOTO(code, &lino, _exit);
_exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
cJSON_Delete(json);
return code;
TAOS_RETURN(code);
}
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {}

11
source/common/src/tdatablock.c
View File

@ -615,6 +615,10 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex)
int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex;
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index);
if (pColInfoData == NULL) {
return 0;
}
if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) {
return 0;
}
@ -1514,7 +1518,7 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
taosArrayDestroy(pBlock->pDataBlock);
pBlock->pDataBlock = NULL;
taosMemoryFreeClear(pBlock->pBlockAgg);
memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
}
@ -2488,7 +2492,10 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha
RandTableName rname = {
.tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname};
buildChildTableName(&rname);
int32_t code = buildChildTableName(&rname);
if(code != TSDB_CODE_SUCCESS){
return code;
}
taosArrayDestroy(tags);
if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) {

76
source/common/src/tdataformat.c
View File

@ -305,11 +305,11 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo
*(int32_t *)(fixed + schema->columns[i].offset) = varlen - fixed - sinfo->tupleFixedSize;
varlen += tPutU32v(varlen, colValArray[colValIndex].value.nData);
if (colValArray[colValIndex].value.nData) {
memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
(void)memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
varlen += colValArray[colValIndex].value.nData;
}
} else {
memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val,
(void)memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val,
tDataTypes[schema->columns[i].type].bytes);
}
} else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL
@ -384,12 +384,12 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c
payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData);
if (colValArray[colValIndex].value.nData > 0) {
memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
(void)memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
}
payloadSize += colValArray[colValIndex].value.nData;
} else {
payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
memcpy(payload + payloadSize, &colValArray[colValIndex].value.val,
(void)memcpy(payload + payloadSize, &colValArray[colValIndex].value.val,
tDataTypes[schema->columns[i].type].bytes);
payloadSize += tDataTypes[schema->columns[i].type].bytes;
}
@ -475,7 +475,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted,
value.nData = infos[iInfo].bind->length[iRow];
value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow;
} else {
memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
(void)memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
infos[iInfo].bind->buffer_length);
}
colVal = COL_VAL_VALUE(infos[iInfo].columnId, value);
@ -509,7 +509,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->cid = pTColumn->colId;
pColVal->value.type = pTColumn->type;
pColVal->flag = CV_FLAG_VALUE;
memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY));
(void)memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY));
return 0;
}
@ -573,7 +573,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = NULL;
}
} else {
memcpy(&pColVal->value.val, pData, pTColumn->bytes);
(void)memcpy(&pColVal->value.val, pData, pTColumn->bytes);
}
}
return 0;
@ -624,7 +624,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset);
pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData);
} else {
memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
(void)memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
}
}
@ -861,7 +861,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.cid = pTColumn->colId;
pIter->cv.value.type = pTColumn->type;
pIter->cv.flag = CV_FLAG_VALUE;
memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY));
(void)memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY));
goto _exit;
}
@ -906,7 +906,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL;
}
} else {
memcpy(&pIter->cv.value.val, pData, pTColumn->bytes);
(void)memcpy(&pIter->cv.value.val, pData, pTColumn->bytes);
}
}
@ -965,7 +965,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL;
}
} else {
memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
(void)memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
}
goto _exit;
}
@ -1288,7 +1288,7 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
key->pks[i].pData = tdata;
key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData);
} else {
memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes);
(void)memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes);
}
}
}
@ -1378,7 +1378,7 @@ FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) {
return 0;
}
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
pDst->ts = pSrc->ts;
pDst->numOfPKs = pSrc->numOfPKs;
@ -1392,12 +1392,10 @@ int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
} else {
pVal->nData = pSrc->pks[i].nData;
ASSERT(pSrc->pks[i].pData != NULL);
memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData);
(void)memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData);
}
}
}
return TSDB_CODE_SUCCESS;
}
// STag ========================================
@ -1528,7 +1526,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
} else {
p = p ? p + n : p;
n += tDataTypes[pTagVal->type].bytes;
if (p) memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes);
if (p) (void)memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes);
}
return n;
@ -1550,7 +1548,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
if (IS_VAR_DATA_TYPE(pTagVal->type)) {
n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData);
} else {
memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes);
(void)memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes);
n += tDataTypes[pTagVal->type].bytes;
}
@ -1661,7 +1659,7 @@ char *tTagValToData(const STagVal *value, bool isJson) {
}
varDataLen(data + typeBytes) = value->nData;
memcpy(varDataVal(data + typeBytes), value->pData, value->nData);
(void)memcpy(varDataVal(data + typeBytes), value->pData, value->nData);
} else {
data = ((char *)&(value->i64)) - typeBytes; // json with type
}
@ -1713,7 +1711,7 @@ bool tTagGet(const STag *pTag, STagVal *pTagVal) {
} else if (c > 0) {
lidx = midx + 1;
} else {
memcpy(pTagVal, &tv, sizeof(tv));
(void)memcpy(pTagVal, &tv, sizeof(tv));
return true;
}
}
@ -1892,7 +1890,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
if (nData) {
code = tRealloc(&pColData->pData, pColData->nData + nData);
if (code) goto _exit;
memcpy(pColData->pData + pColData->nData, pData, nData);
(void)memcpy(pColData->pData + pColData->nData, pData, nData);
pColData->nData += nData;
}
} else {
@ -1900,7 +1898,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes);
if (code) goto _exit;
if (pData) {
memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]);
(void)memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]);
} else {
memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]);
}
@ -2594,7 +2592,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo
}
value.pData = pColData->pData + pColData->aOffset[iVal];
} else {
memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes);
(void)memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes);
}
*pColVal = COL_VAL_VALUE(pColData->cid, value);
}
@ -2692,7 +2690,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal));
(void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal));
break;
case (HAS_VALUE | HAS_NULL | HAS_NONE):
pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal));
@ -2700,7 +2698,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal));
(void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal));
break;
default:
pColData->pBitMap = NULL;
@ -2714,7 +2712,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2);
(void)memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2);
} else {
pColData->aOffset = NULL;
}
@ -2727,7 +2725,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
goto _exit;
}
memcpy(pColData->pData, pColDataFrom->pData, pColData->nData);
(void)memcpy(pColData->pData, pColDataFrom->pData, pColData->nData);
} else {
pColData->pData = NULL;
}
@ -3119,10 +3117,10 @@ static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SCo
pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData;
}
memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow],
(void)memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow],
nData);
} else {
memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow],
(void)memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow],
&pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]);
}
return code;
@ -3361,7 +3359,7 @@ static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd /
ASSERT(0);
} else {
if (iv != iStart) {
memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart],
(void)memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart],
&pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]);
}
memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)],
@ -3579,11 +3577,11 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) {
case (HAS_NULL | HAS_NONE):
case (HAS_VALUE | HAS_NONE):
case (HAS_VALUE | HAS_NULL):
if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal));
if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal));
n += BIT1_SIZE(pColData->nVal);
break;
case (HAS_VALUE | HAS_NULL | HAS_NONE):
if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal));
if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal));
n += BIT2_SIZE(pColData->nVal);
break;
default:
@ -3593,14 +3591,14 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) {
// value
if (pColData->flag & HAS_VALUE) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2);
if (pBuf) (void)memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2);
n += (pColData->nVal << 2);
n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData);
if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData);
if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData);
n += pColData->nData;
} else {
if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData);
if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData);
n += pColData->nData;
}
}
@ -4348,7 +4346,7 @@ int32_t tCompressData(void *input, // input
ASSERT(outputSize >= extraSizeNeeded);
if (info->cmprAlg == NO_COMPRESSION) {
memcpy(output, input, info->originalSize);
(void)memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize;
} else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local;
@ -4385,7 +4383,7 @@ int32_t tCompressData(void *input, // input
} else {
DEFINE_VAR(info->cmprAlg)
if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) {
memcpy(output, input, info->originalSize);
(void)memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize;
return 0;
}
@ -4431,7 +4429,7 @@ int32_t tDecompressData(void *input, // input
if (info->cmprAlg == NO_COMPRESSION) {
ASSERT(info->compressedSize == info->originalSize);
memcpy(output, input, info->compressedSize);
(void)memcpy(output, input, info->compressedSize);
} else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local;
@ -4468,7 +4466,7 @@ int32_t tDecompressData(void *input, // input
} else {
DEFINE_VAR(info->cmprAlg);
if (l1 == L1_DISABLED && l2 == L2_DISABLED) {
memcpy(output, input, info->compressedSize);
(void)memcpy(output, input, info->compressedSize);
return 0;
}
SBuffer local;

2
source/common/src/tglobal.c
View File

@ -504,7 +504,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1;

167
source/common/src/tmisce.c
View File

@ -15,9 +15,9 @@
#define _DEFAULT_SOURCE
#include "tmisce.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tjson.h"
#include "tdatablock.h"
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = 0;
@ -34,18 +34,26 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = tsServerPort;
}
if (pEp->port <= 0) {
return TSDB_CODE_INVALID_PARA;
}
return 0;
}
void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) {
int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) {
if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) {
return;
return TSDB_CODE_INVALID_PARA;
}
int32_t index = pEpSet->numOfEps;
if (index >= sizeof(pEpSet->eps) / sizeof(pEpSet->eps[0])) {
return TSDB_CODE_OUT_OF_RANGE;
}
tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn));
pEpSet->eps[index].port = port;
pEpSet->numOfEps += 1;
return 0;
}
bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) {
@ -131,75 +139,96 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet) {
return ep;
}
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t bufLen) {
int len = snprintf(pBuf, bufLen, "epset:{");
if (len < 0) {
return -1;
}
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) {
int32_t ret = 0;
int32_t nwrite = 0;
for (int _i = 0; (_i < pEpSet->numOfEps) && (bufLen > len); _i++) {
nwrite = snprintf(pBuf + nwrite, cap, "epset:{");
if (nwrite <= 0 || nwrite >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
cap -= nwrite;
for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) {
int32_t ret = 0;
if (_i == pEpSet->numOfEps - 1) {
ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} else {
ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
}
if (ret < 0) {
return -1;
if (ret <= 0 || ret >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
len += ret;
nwrite += ret;
cap -= ret;
}
if (len < bufLen) {
/*len += */snprintf(pBuf + len, bufLen - len, "}, inUse:%d", pEpSet->inUse);
if (cap <= 0) {
return TSDB_CODE_OUT_OF_BUFFER;
}
ret = snprintf(pBuf + nwrite, cap, "}, inUse:%d", pEpSet->inUse);
if (ret <= 0 || ret >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
return TSDB_CODE_SUCCESS;
}
int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) {
SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return -1;
int32_t code = 0;
SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
char tmp[4096] = {0};
tjsonAddDoubleToObject(pJson, "reportVersion", 1);
TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "reportVersion", 1), NULL, _exit);
tjsonAddIntegerToObject(pJson, "clusterId", clusterId);
tjsonAddIntegerToObject(pJson, "startTime", startTime);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "clusterId", clusterId), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "startTime", startTime), NULL, _exit);
// Do NOT invoke the taosGetFqdn here.
// this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked
// environment. The lock operation by taosGetFqdn may cause this program deadlock.
tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn), NULL, _exit);
tjsonAddIntegerToObject(pJson, "pid", taosGetPId());
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "pid", taosGetPId()), NULL, _exit);
taosGetAppName(tmp, NULL);
tjsonAddStringToObject(pJson, "appName", tmp);
code = taosGetAppName(tmp, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "appName", tmp), NULL, _exit);
if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) {
tjsonAddStringToObject(pJson, "os", tmp);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "os", tmp), NULL, _exit);
} else {
// do nothing
}
float numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) {
tjsonAddStringToObject(pJson, "cpuModel", tmp);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "cpuModel", tmp), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores), NULL, _exit);
} else {
tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores);
TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores), NULL, _exit);
}
snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
int32_t nBytes = snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
if (nBytes <= 9 || nBytes >= sizeof(tmp)) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_RANGE, NULL, _exit);
}
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit);
tjsonAddStringToObject(pJson, "version", version);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo);
tjsonAddStringToObject(pJson, "gitInfo", gitinfo);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", version), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", buildinfo), NULL, _exit);
tjsonAddIntegerToObject(pJson, "crashSig", signum);
tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs());
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", gitinfo), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit);
#ifdef _TD_DARWIN_64
taosLogTraceToBuf(tmp, sizeof(tmp), 4);
@ -209,26 +238,47 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t
taosLogTraceToBuf(tmp, sizeof(tmp), 8);
#endif
tjsonAddStringToObject(pJson, "stackInfo", tmp);
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "stackInfo", tmp), NULL, _exit);
char* pCont = tjsonToString(pJson);
if (pCont == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_GOTO(code, NULL, _exit);
goto _exit;
}
tjsonDelete(pJson);
*pMsg = pCont;
return TSDB_CODE_SUCCESS;
pJson = NULL;
_exit:
tjsonDelete(pJson);
TAOS_RETURN(code);
}
int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
SConfig* pConf = taosGetCfg();
int32_t code = 0;
SConfig* pConf = taosGetCfg();
if (pConf == NULL) {
return TSDB_CODE_INVALID_CFG;
}
int32_t numOfRows = 0;
int32_t col = startCol;
SConfigItem* pItem = NULL;
SConfigIter* pIter = NULL;
blockDataEnsureCapacity(pBlock, cfgGetSize(pConf));
SConfigIter* pIter = cfgCreateIter(pConf);
int8_t locked = 0;
TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit);
pIter = cfgCreateIter(pConf);
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
cfgLock(pConf);
locked = 1;
while ((pItem = cfgNextIter(pIter)) != NULL) {
col = startCol;
@ -236,29 +286,44 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
// GRANT_CFG_SKIP;
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, name, false);
if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit);
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
int32_t valueLen = 0;
cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
varDataSetLen(value, valueLen);
pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, value, false);
if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, value, false), NULL, _exit);
char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0};
cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen);
varDataSetLen(scope, valueLen);
pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, scope, false);
if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit);
numOfRows++;
}
cfgUnLock(pConf);
pBlock->info.rows = numOfRows;
_exit:
if (locked) cfgUnLock(pConf);
cfgDestroyIter(pIter);
return TSDB_CODE_SUCCESS;
TAOS_RETURN(code);
}

13
source/common/src/tname.c
View File

@ -293,17 +293,19 @@ static int compareKv(const void* p1, const void* p2) {
/*
* use stable name and tags to grearate child table name
*/
void buildChildTableName(RandTableName* rName) {
int32_t buildChildTableName(RandTableName* rName) {
SStringBuilder sb = {0};
taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen);
if (sb.buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return;
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArraySort(rName->tags, compareKv);
for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) {
taosStringBuilderAppendChar(&sb, ',');
SSmlKv* tagKv = taosArrayGet(rName->tags, j);
if(tagKv == NULL) {
return TSDB_CODE_SML_INVALID_DATA;
}
taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
taosStringBuilderAppendChar(&sb, '=');
if (IS_VAR_DATA_TYPE(tagKv->type)) {
@ -323,8 +325,9 @@ void buildChildTableName(RandTableName* rName) {
rName->ctbShortName[0] = 't';
rName->ctbShortName[1] = '_';
for (int i = 0; i < 16; i++) {
sprintf(temp, "%02x", context.digest[i]);
strcat(rName->ctbShortName, temp);
(void)sprintf(temp, "%02x", context.digest[i]);
(void)strcat(rName->ctbShortName, temp);
}
taosStringBuilderDestroy(&sb);
return TSDB_CODE_SUCCESS;
}

2
source/dnode/vnode/inc/vnode.h
View File

@ -167,7 +167,7 @@ void tsdbReaderClose2(STsdbReader *pReader);
int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext);
int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA);
void tsdbReleaseDataBlock2(STsdbReader *pReader);
SSDataBlock *tsdbRetrieveDataBlock2(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList);
int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond);
int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle);

6
source/dnode/vnode/src/inc/tsdb.h
View File

@ -909,9 +909,9 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree);
bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree *pMTree);
SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols);
void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void * destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo);
void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
// tsdbCache ==============================================================================================
typedef enum {

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -54,7 +54,6 @@ extern "C" {
#endif
typedef struct SVnodeInfo SVnodeInfo;
typedef struct SMeta SMeta;
typedef struct SSma SSma;
typedef struct STsdb STsdb;
typedef struct STQ STQ;
@ -153,7 +152,6 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode);
void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive);
// meta
typedef struct SMCtbCursor SMCtbCursor;
typedef struct SMStbCursor SMStbCursor;
typedef struct STbUidStore STbUidStore;

41
source/dnode/vnode/src/tq/tqSink.c
View File

@ -46,6 +46,7 @@ static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid,
static SArray* createDefaultTagColName();
static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule);
static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo);
int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
const char* pIdStr, bool newSubTableRule) {
@ -270,6 +271,14 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayPush(reqs.pArray, pCreateTbReq);
STableSinkInfo* pInfo = NULL;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo);
if (!alreadyCached) {
code = doCreateSinkInfo(pCreateTbReq->name, &pInfo);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, pTask->id.idStr);
}
tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name);
}
@ -634,6 +643,18 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
return TSDB_CODE_SUCCESS;
}
int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo) {
int32_t nameLen = strlen(pDstTableName);
(*pInfo) = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1);
if (*pInfo == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pInfo)->name.len = nameLen;
memcpy((*pInfo)->name.data, pDstTableName, nameLen);
return TSDB_CODE_SUCCESS;
}
int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName,
SSubmitTbData* pTableData) {
uint64_t groupId = pDataBlock->info.id.groupId;
@ -670,22 +691,15 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) &&
!alreadyAddGroupId(dstTableName, groupId) && groupId != 0) {
tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName);
if(pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER){
if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) {
buildCtbNameAddGroupId(NULL, dstTableName, groupId);
}else if(pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) {
} else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) {
buildCtbNameAddGroupId(stbFullName, dstTableName, groupId);
}
}
}
int32_t nameLen = strlen(dstTableName);
pTableSinkInfo = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1);
if (pTableSinkInfo == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pTableSinkInfo->name.len = nameLen;
memcpy(pTableSinkInfo->name.data, dstTableName, nameLen);
int32_t code = doCreateSinkInfo(dstTableName, &pTableSinkInfo);
tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName);
}
@ -693,7 +707,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
pTableData->uid = pTableSinkInfo->uid;
if (pTableData->uid == 0) {
tqTrace("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id);
tqTrace("s-task:%s cached tableInfo:%s uid is invalid, acquire it from meta", id, pTableSinkInfo->name.data);
return doWaitForDstTableCreated(pVnode, pTask, pTableSinkInfo, dstTableName, &pTableData->uid);
} else {
tqTrace("s-task:%s set the dstTable uid from cache:%" PRId64, id, pTableData->uid);
@ -929,11 +943,6 @@ bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) {
}
int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) {
if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) {
taosMemoryFreeClear(pTableSinkInfo); // too many items, failed to cache it
return TSDB_CODE_FAILED;
}
int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pTableSinkInfo);

7
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -2579,7 +2579,7 @@ typedef struct {
static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) {
int32_t code = 0;
pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, NULL);
destroySttBlockReader(pr->pLDataIterArray, NULL);
pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
SMergeTreeConf conf = {
@ -3212,7 +3212,10 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
pIter->pMemDelData = NULL;
loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer);
code = loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
pIter->idx = (SBlockIdx){.suid = suid, .uid = uid};

2
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -203,7 +203,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf
pReader->pTableList = pTableIdList;
pReader->numOfTables = numOfTables;
pReader->lastTs = INT64_MIN;
pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, NULL);
destroySttBlockReader(pReader->pLDataIterArray, NULL);
pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
return TSDB_CODE_SUCCESS;

35
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -23,11 +23,12 @@
static void tLDataIterClose2(SLDataIter *pIter);
// SLDataIter =================================================
SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) {
int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo) {
*pInfo = NULL;
SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo));
if (pLoadInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
pLoadInfo->blockData[0].sttBlockIndex = -1;
@ -37,26 +38,29 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList,
int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data);
if (code) {
terrno = code;
taosMemoryFreeClear(pLoadInfo);
return code;
}
code = tBlockDataCreate(&pLoadInfo->blockData[1].data);
if (code) {
terrno = code;
taosMemoryFreeClear(pLoadInfo);
return code;
}
pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk));
if (pLoadInfo->aSttBlk == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFreeClear(pLoadInfo);
return NULL;
return code;
}
pLoadInfo->pSchema = pSchema;
pLoadInfo->colIds = colList;
pLoadInfo->numOfCols = numOfCols;
return pLoadInfo;
*pInfo = pLoadInfo;
return code;
}
static void freeItem(void* pValue) {
@ -66,9 +70,9 @@ static void freeItem(void* pValue) {
}
}
void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
if (pLoadInfo == NULL) {
return NULL;
return;
}
pLoadInfo->currentLoadBlockIndex = 1;
@ -94,7 +98,6 @@ void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
taosArrayDestroy(pLoadInfo->aSttBlk);
taosMemoryFree(pLoadInfo);
return NULL;
}
void destroyLDataIter(SLDataIter *pIter) {
@ -103,9 +106,9 @@ void destroyLDataIter(SLDataIter *pIter) {
taosMemoryFree(pIter);
}
void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) {
void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) {
if (pLDataIterArray == NULL) {
return NULL;
return;
}
int32_t numOfLevel = taosArrayGetSize(pLDataIterArray);
@ -132,7 +135,6 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa
}
taosArrayDestroy(pLDataIterArray);
return NULL;
}
// choose the unpinned slot to load next data block
@ -914,9 +916,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF
}
if (pLoadInfo == NULL) {
pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols);
if (pLoadInfo == NULL) {
code = terrno;
code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pLoadInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
}

761
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

File diff suppressed because it is too large Load Diff

112
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -112,22 +112,28 @@ void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {
taosArrayDestroy(pBuf->pData);
}
void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) {
int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) {
*pInfo = NULL;
int32_t bucketIndex = index / pBuf->numPerBucket;
char** pBucket = taosArrayGet(pBuf->pData, bucketIndex);
return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo);
}
STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) {
STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (p == NULL || *p == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
int32_t size = tSimpleHashGetSize(pTableMap);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);
return NULL;
if (pBucket == NULL) {
return TSDB_CODE_FAILED;
}
return *p;
*pInfo = (STableBlockScanInfo*)((*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo));
return TSDB_CODE_SUCCESS;
}
int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) {
*pInfo = *(STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (pInfo == NULL) {
int32_t size = tSimpleHashGetSize(pTableMap);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);
return TSDB_CODE_INVALID_PARA;
}
return TSDB_CODE_SUCCESS;
}
int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) {
@ -146,17 +152,17 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
}
case TSDB_DATA_TYPE_INT:{
int32_t min = INT32_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_SMALLINT:{
int16_t min = INT16_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_TINYINT:{
int8_t min = INT8_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_UTINYINT:
@ -245,20 +251,27 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH
initLastProcKey(pScanInfo, pReader);
pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
int32_t code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid,
pScanInfo->lastProcKey.ts, pReader->idStr);
return TSDB_CODE_SUCCESS;
return code;
}
// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model
SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables) {
int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) {
int32_t code = 0;
*pHashObj = NULL;
// allocate buffer in order to load data blocks from file
// todo use simple hash instead, optimize the memory consumption
SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (pTableMap == NULL) {
return NULL;
return terrno;
}
int64_t st = taosGetTimestampUs();
@ -267,7 +280,7 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
if (pUidList->tableUidList == NULL) {
tSimpleHashCleanup(pTableMap);
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
pUidList->currentIndex = 0;
@ -275,8 +288,16 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
for (int32_t j = 0; j < numOfTables; ++j) {
pUidList->tableUidList[j] = idList[j].uid;
STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j);
initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);
STableBlockScanInfo* pScanInfo = NULL;
code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo);
if (code != TSDB_CODE_SUCCESS) {
break;
}
code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);
if (code != TSDB_CODE_SUCCESS) {
break;
}
}
taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc);
@ -286,7 +307,8 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
(sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList,
pTsdbReader->idStr);
return pTableMap;
*pHashObj = pTableMap;
return code;
}
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {
@ -391,11 +413,13 @@ void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray
pIter->pBrinBlockList = pList;
}
SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {
*pRecord = NULL;
if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) {
pIter->blockIndex += 1;
if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) {
return NULL;
return TSDB_CODE_FAILED;
}
pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);
@ -404,7 +428,7 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));
return NULL;
return TSDB_CODE_FAILED;
}
pIter->recordIndex = -1;
@ -412,7 +436,9 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
pIter->recordIndex += 1;
tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record);
return &pIter->record;
*pRecord = &pIter->record;
return TSDB_CODE_SUCCESS;
}
void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); }
@ -670,7 +696,13 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
STombRecord record = {0};
uint64_t uid = pReader->status.uidList.tableUidList[*j];
STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr);
STableBlockScanInfo* pScanInfo = NULL;
code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
}
@ -702,7 +734,11 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
}
uid = pReader->status.uidList.tableUidList[*j];
pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr);
code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
}
@ -806,9 +842,12 @@ int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileRead
return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false);
}
void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {
int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {
if (*ppMemDelData == NULL) {
*ppMemDelData = taosArrayInit(4, sizeof(SDelData));
if (*ppMemDelData == NULL) {
return TSDB_CODE_SUCCESS;
}
}
SArray* pMemDelData = *ppMemDelData;
@ -836,6 +875,8 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT
p = p->pNext;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
@ -960,6 +1001,7 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet)
int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf,
const char* pstr) {
int32_t numOfRows = 0;
int32_t code = 0;
// no data exists, go to end
int32_t numOfLevels = pFileSet->lvlArr->size;
@ -984,7 +1026,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
conf.file[0] = *pSttLevel->fobjArr->data[i]->f;
const char* pName = pSttLevel->fobjArr->data[i]->fname;
int32_t code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);
code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr);
continue;
@ -992,8 +1034,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
}
if (pIter->pBlockLoadInfo == NULL) {
pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols);
if (pIter->pBlockLoadInfo == NULL) {
code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to create block load info, code: out of memory, %s", pstr);
continue;
}
@ -1001,7 +1043,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
// load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file
TStatisBlkArray* pStatisBlkArray = NULL;
int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);
code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr);
continue;

41
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -315,32 +315,31 @@ typedef struct SBrinRecordIter {
} SBrinRecordIter;
int32_t uidComparFunc(const void* p1, const void* p2);
int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id);
STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id);
SSHashObj* createDataBlockScanInfo(STsdbReader* pReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables);
int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,
STsdbReader* pReader);
void clearBlockScanInfo(STableBlockScanInfo* p);
void destroyAllBlockScanInfo(SSHashObj* pTableMap);
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step);
void cleanupInfoForNextFileset(SSHashObj* pTableMap);
int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables);
void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf);
void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index);
int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj);
int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,
STsdbReader* pReader);
void clearBlockScanInfo(STableBlockScanInfo* p);
void destroyAllBlockScanInfo(SSHashObj* pTableMap);
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step);
void cleanupInfoForNextFileset(SSHashObj* pTableMap);
int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables);
void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf);
int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pRes);
// brin records iterator
void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList);
SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter);
void clearBrinBlockIter(SBrinRecordIter* pIter);
void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList);
int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord);
void clearBrinBlockIter(SBrinRecordIter* pIter);
// initialize block iterator API
int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList);
bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr);
// load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time)
void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver);
int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver);
int32_t loadDataFileTombDataForAll(STsdbReader* pReader);
int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo);
int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
@ -358,10 +357,10 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2);
int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc);
void clearRowKey(SRowKey* pKey);
bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp);
void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk);
void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree);
void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk);
bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp);
int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk);
void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree);
void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk);
typedef struct {
SArray* pTombData;

4
source/libs/audit/src/auditMain.c
View File

@ -32,8 +32,8 @@ char* tsAuditBatchUri = "/audit-batch";
int32_t auditInit(const SAuditCfg *pCfg) {
tsAudit.cfg = *pCfg;
tsAudit.records = taosArrayInit(0, sizeof(SAuditRecord *));
taosThreadMutexInit(&tsAudit.lock, NULL);
return 0;
if(tsAudit.records == NULL) return TSDB_CODE_OUT_OF_MEMORY;
return taosThreadMutexInit(&tsAudit.lock, NULL);
}
static FORCE_INLINE void auditDeleteRecord(SAuditRecord * record) {

2
source/libs/executor/inc/executorInt.h
View File

@ -929,7 +929,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i
SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore,
SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr,
SStorageAPI* pApi, int32_t tsIndex);
void initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic);
void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins);
void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);

1
source/libs/executor/inc/streamexecutorInt.h
View File

@ -20,6 +20,7 @@ extern "C" {
#endif
#include "executorInt.h"
#include "tutil.h"
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);

18
source/libs/executor/src/groupoperator.c
View File

@ -17,6 +17,7 @@
#include "function.h"
#include "os.h"
#include "tname.h"
#include "tutil.h"
#include "tdatablock.h"
#include "tmsg.h"
@ -1357,11 +1358,13 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
taosMemoryFreeClear(param);
}
void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) {
int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI;
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
return;
return code;
}
SStreamScanInfo* pScanInfo = downstream->info;
@ -1369,8 +1372,9 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup
pScanInfo->pPartScalarSup = pExpr;
pScanInfo->pPartTbnameSup = pTbnameExpr;
if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen);
code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
}
return code;
}
SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) {
@ -1421,6 +1425,7 @@ void freePartItem(void* ptr) {
SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -1511,14 +1516,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup);
code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
return pOperator;
_error:
pTaskInfo->code = code;
destroyStreamPartitionOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator);
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}

88
source/libs/executor/src/scanoperator.c
View File

@ -388,9 +388,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1;
SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
if (p == NULL) {
return terrno;
SSDataBlock* p = NULL;
int32_t code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, &p, NULL);
if (p == NULL || code != TSDB_CODE_SUCCESS) {
return code;
}
ASSERT(p == pBlock);
@ -1351,7 +1352,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
}
if (hasNext) {
/*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL);
SSDataBlock* p = NULL;
code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, &p, NULL);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
}
@ -2477,27 +2483,54 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl
}
}
int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) {
int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo);
len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup);
*pBuff = taosMemoryCalloc(1, len);
void* buf = *pBuff;
encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup);
pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo);
return len;
int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff, int32_t* pLen) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
int32_t len = 0;
code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len);
TSDB_CHECK_CODE(code, lino, _end);
len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup);
*pBuff = taosMemoryCalloc(1, len);
if (!(*pBuff)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
void* buf = *pBuff;
encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup);
int32_t tmp = 0;
code = pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo, &tmp);
TSDB_CHECK_CODE(code, lino, _end);
*pLen = len;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (!pInfo->pState) {
return;
}
if (needSaveStreamOperatorInfo(&pInfo->basic)) {
void* pBuf = NULL;
int32_t len = streamScanOperatorEncode(pInfo, &pBuf);
int32_t len = 0;
code = streamScanOperatorEncode(pInfo, &pBuf, &len);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
taosMemoryFree(pBuf);
saveStreamOperatorStateComplete(&pInfo->basic);
}
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
}
// other properties are recovered from the execution plan
@ -2929,8 +2962,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
}
SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL);
if (pBlock == NULL) {
SSDataBlock* pBlock = NULL;
code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, &pBlock, NULL);
if (pBlock == NULL || code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
@ -3094,17 +3128,35 @@ static void destroyStreamScanOperatorInfo(void* param) {
}
void streamScanReleaseState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamScanInfo* pInfo = pOperator->info;
void* pBuff = NULL;
if (!pInfo->pState) {
return;
}
if (!pInfo->pUpdateInfo) {
return;
}
int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo);
void* pBuff = taosMemoryCalloc(1, len);
pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo);
int32_t len = 0;
code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len);
TSDB_CHECK_CODE(code, lino, _end);
pBuff = taosMemoryCalloc(1, len);
if (!pBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
int32_t tmp = 0;
code = pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo, &tmp);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosMemoryFree(pBuff);
}

90
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -68,9 +68,7 @@ void destroyStreamCountAggOperatorInfo(void* param) {
taosMemoryFreeClear(param);
}
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) {
return pAggSup->windowCount != pAggSup->windowSliding;
}
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; }
void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
SBuffInfo* pBuffInfo) {
@ -89,13 +87,14 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
ASSERT(pBuffInfo->pCur);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
(void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
}
} else {
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount);
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) {
@ -107,15 +106,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
pBuffInfo->rebuildWindow = true;
}
} else {
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(
pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size);
}
if (code == TSDB_CODE_SUCCESS) {
pCurWin->winInfo.isOutput = true;
}
pCurWin->pWindowCount=
(COUNT_TYPE*) ((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE)));
pCurWin->pWindowCount =
(COUNT_TYPE*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE)));
if (*pCurWin->pWindowCount == pAggSup->windowCount) {
pBuffInfo->rebuildWindow = true;
@ -133,7 +133,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI
int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool* pRebuild) {
SSessionKey sWinKey = pWinInfo->winInfo.sessionWin;
int32_t num = 0;
int32_t num = 0;
for (int32_t i = start; i < rows; i++) {
if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) {
num++;
@ -181,7 +181,7 @@ void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, ESt
pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey);
}
SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0);
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0);
if (code != TSDB_CODE_SUCCESS) {
pAggSup->stateStore.streamStateFreeCur(pCur);
return;
@ -239,7 +239,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2);
SStreamStateCur* pCur = NULL;
COUNT_TYPE slidingRows = 0;
COUNT_TYPE slidingRows = 0;
for (int32_t i = 0; i < rows;) {
if (pInfo->ignoreExpiredData &&
@ -269,7 +269,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
}
getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range);
range.win.skey = TMIN(startTsCols[i], range.win.skey);
range.win.ekey = TMAX(startTsCols[rows-1], range.win.ekey);
range.win.ekey = TMAX(startTsCols[rows - 1], range.win.ekey);
uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL);
break;
@ -289,8 +289,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
code = saveResult(curWin.winInfo, pStUpdated);
if (code != TSDB_CODE_SUCCESS) {
qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo),
tstrerror(code));
qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
break;
}
}
@ -397,13 +396,14 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
int32_t mapSize = 0;
buf = taosDecodeFixedI32(buf, &mapSize);
for (int32_t i = 0; i < mapSize; i++) {
SSessionKey key = {0};
SSessionKey key = {0};
SCountWindowInfo curWin = {0};
buf = decodeSSessionKey(buf, &key);
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo,
sizeof(SResultWindowInfo));
}
// 2.twAggSup
@ -441,9 +441,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
uint64_t* gpDatas = (uint64_t*)pGroupCol->pData;
SRowBuffPos* pPos = NULL;
int32_t size = 0;
int32_t size = 0;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]};
SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]};
SStreamStateCur* pCur = NULL;
if (isSlidingCountWindow(pAggSup)) {
pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount);
@ -452,7 +452,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
}
while (1) {
SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size);
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void**)&pPos, &size);
if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) {
pAggSup->stateStore.streamStateFreeCur(pCur);
break;
@ -482,7 +482,7 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr
SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]};
while (1) {
SSessionKey curWin = {0};
int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin);
int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin);
if (code == TSDB_CODE_FAILED) {
break;
}
@ -511,11 +511,11 @@ void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHa
}
static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pOperator->exprSupp;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExprSupp* pSup = &pOperator->exprSupp;
SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
@ -635,7 +635,7 @@ void streamCountReloadState(SOperatorInfo* pOperator) {
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME,
strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size);
TSKEY ts = *(TSKEY*)pBuf;
TSKEY ts = *(TSKEY*)pBuf;
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
taosMemoryFree(pBuf);
@ -650,11 +650,13 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) {
SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode;
int32_t numOfCols = 0;
int32_t code = TSDB_CODE_OUT_OF_MEMORY;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pOperator->pTaskInfo = pTaskInfo;
@ -664,18 +666,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
}
SExprSupp* pExpSup = &pOperator->exprSupp;
SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
pInfo->twAggSup = (STimeWindowAggSupp){
.waterMark = pCountNode->window.watermark,
@ -686,12 +684,11 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
};
pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId;
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0,
pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle,
&pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState,
sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
TSDB_CHECK_CODE(code, lino, _error);
pInfo->streamAggSup.windowCount = pCountNode->windowCount;
pInfo->streamAggSup.windowSliding = pCountNode->windowSliding;
@ -709,7 +706,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) {
goto _error;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
@ -735,8 +733,12 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
}
return pOperator;
@ -747,6 +749,6 @@ _error:
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}

112
source/libs/executor/src/streameventwindowoperator.c
View File

@ -28,9 +28,9 @@
#include "tlog.h"
#include "ttime.h"
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
typedef struct SEventWinfowFlag {
bool startFlag;
@ -86,10 +86,11 @@ void destroyStreamEventOperatorInfo(void* param) {
void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) {
char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize);
pWinInfo->pWinFlag = (SEventWinfowFlag*) pFlagInfo;
pWinInfo->pWinFlag = (SEventWinfowFlag*)pFlagInfo;
}
void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, SEventWindowInfo* pWinInfo) {
void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos,
SEventWindowInfo* pWinInfo) {
pWinInfo->winInfo.sessionWin = *pKey;
pWinInfo->winInfo.pStatePos = pPos;
setEventWindowFlag(pAggSup, pWinInfo);
@ -122,22 +123,23 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts;
pCurWin->winInfo.sessionWin.win.ekey = ts;
SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin);
SSessionKey leftWinKey = {.groupId = groupId};
void* pVal = NULL;
int32_t len = 0;
SStreamStateCur* pCur =
pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin);
SSessionKey leftWinKey = {.groupId = groupId};
void* pVal = NULL;
int32_t len = 0;
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win) ) {
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) {
bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0);
setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin);
if(inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag) ) {
if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) {
pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin);
goto _end;
}
}
pAggSup->stateStore.streamStateFreeCur(pCur);
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
SSessionKey rightWinKey = {.groupId = groupId};
SSessionKey rightWinKey = {.groupId = groupId};
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len);
bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) {
@ -149,7 +151,7 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
}
}
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
pCurWin->pWinFlag->startFlag = start;
@ -164,7 +166,7 @@ _end:
if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pNextWinKey);
}
if(pCurWin->winInfo.pStatePos->needFree) {
if (pCurWin->winInfo.pStatePos->needFree) {
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);
}
pAggSup->stateStore.streamStateFreeCur(pCur);
@ -173,14 +175,14 @@ _end:
}
int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey,
TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows,
SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) {
TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start,
SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) {
*pRebuild = false;
if (!pWinInfo->pWinFlag->startFlag && !(starts[start]) ) {
if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) {
return 1;
}
TSKEY maxTs = INT64_MAX;
TSKEY maxTs = INT64_MAX;
STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win;
if (pWinInfo->pWinFlag->endFlag) {
maxTs = pWin->ekey + 1;
@ -219,7 +221,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));
if (ends[i]) {
if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey ) {
if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) {
*pRebuild = true;
}
return i + 1 - start;
@ -230,26 +232,28 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool addGap) {
SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
int32_t winNum = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
int32_t winNum = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
while (1) {
if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) {
break;
}
SEventWindowInfo nextWinInfo = {0};
getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo);
if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) {
if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) ||
!inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) {
releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);
break;
}
setEventWindowFlag(pAggSup, &nextWinInfo);
compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, pStDeleted, false);
compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated,
pStDeleted, false);
pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag;
winNum++;
}
@ -275,7 +279,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t winRows = 0;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SColumnInfoData* pColStart = NULL;
SColumnInfoData* pColEnd = NULL;
SColumnInfoData* pColEnd = NULL;
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
@ -290,7 +294,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
return;
}
SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock};
SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pStartCondInfo, &paramStart);
if (code != TSDB_CODE_SUCCESS) {
qError("set data from start slotId error.");
@ -299,7 +304,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t statusStart = 0;
filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart);
SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock};
SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pEndCondInfo, &paramEnd);
if (code != TSDB_CODE_SUCCESS) {
qError("set data from end slotId error.");
@ -320,20 +326,23 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t winIndex = 0;
bool allEqual = true;
SEventWindowInfo curWin = {0};
SSessionKey nextWinKey = {0};
setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, &nextWinKey);
SSessionKey nextWinKey = {0};
setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin,
&nextWinKey);
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
bool rebuild = false;
winRows = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i,
pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild);
winRows =
updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild);
ASSERT(winRows >= 1);
if (rebuild) {
uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin);
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) {
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) &&
!isWindowIncomplete(&curWin)) {
saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
}
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
@ -603,7 +612,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) {
char* pBuff = taosMemoryCalloc(1, resSize);
memcpy(pBuff, pInfo->historyWins->pData, winSize);
memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
qDebug("===stream=== event window operator relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins));
qDebug("===stream=== event window operator relase state. save result count:%d",
(int32_t)taosArrayGetSize(pInfo->historyWins));
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME,
strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize);
pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);
@ -647,7 +657,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,
pSeKeyBuf[i].groupId, i);
getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo);
//event window has been deleted
// event window has been deleted
if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) {
continue;
}
@ -659,7 +669,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId);
curInfo.winInfo.sessionWin.groupId);
if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {
saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
}
@ -693,6 +703,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode;
int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -783,20 +794,18 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
return pOperator;
@ -804,5 +813,6 @@ _error:
destroyStreamEventOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}

192
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -28,14 +28,18 @@
#include "tlog.h"
#include "ttime.h"
#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
#define IS_NORMAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
#define IS_NORMAL_INTERVAL_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_SESSION_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
#define DEAULT_DELETE_MARK INT64_MAX
#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState"
@ -263,14 +267,16 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
if (chIds) {
int32_t childId = getChildIndex(pBlock);
if (pInvalidWins) {
qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId);
qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts,
winRes.groupId, childId);
taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0);
}
SArray* chArray = *(void**)chIds;
int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);
if (index != -1) {
qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, winGpId, childId);
qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey,
winGpId, childId);
getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
continue;
}
@ -390,7 +396,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
}
void destroyFlusedPos(void* pRes) {
SRowBuffPos* pPos = (SRowBuffPos*) pRes;
SRowBuffPos* pPos = (SRowBuffPos*)pRes;
if (!pPos->needFree && !pPos->pRowBuff) {
taosMemoryFreeClear(pPos->pKey);
taosMemoryFree(pPos);
@ -488,8 +494,11 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt
pScanInfo->windowSup.parentType = type;
pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;
if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo =
pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen);
int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate,
pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at %d since %s", __func__, __LINE__, tstrerror(code));
}
}
pScanInfo->interval = pInfo->interval;
@ -624,17 +633,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
blockDataUpdateTsWindow(pBlock, 0);
}
static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins,
int32_t numOfCh, SOperatorInfo* pOperator) {
static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval,
SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* tsData = (TSKEY*)pStartCol->pData;
SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
TSKEY* tsEndData = (TSKEY*)pEndCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
int32_t chId = getChildIndex(pBlock);
bool res = false;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* tsData = (TSKEY*)pStartCol->pData;
SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
TSKEY* tsEndData = (TSKEY*)pEndCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
int32_t chId = getChildIndex(pBlock);
bool res = false;
for (int32_t i = 0; i < pBlock->info.rows; i++) {
TSKEY winTs = tsData[i];
while (winTs <= tsEndData[i]) {
@ -650,7 +659,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
// pull data is over
taosArrayDestroy(chArray);
taosHashRemove(pMap, &winRes, sizeof(SWinKey));
res =true;
res = true;
qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts);
void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey));
@ -663,7 +672,8 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
.calWin.skey = nextWin.skey,
.calWin.ekey = nextWin.skey};
// add pull data request
qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", winRes.ts, winRes.groupId, numOfCh);
qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d",
winRes.ts, winRes.groupId, numOfCh);
if (IS_MID_INTERVAL_OP(pOperator)) {
SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;
taosArrayPush(pInfo->pMidPullDatas, &winRes);
@ -671,7 +681,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
taosArrayPush(pInfo->pDelWins, &winRes);
addPullWindow(pMap, &winRes, numOfCh);
if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0);
tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);
}
qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh);
}
@ -690,7 +700,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i
for (int32_t i = 0; i < size; i++) {
SWinKey* winKey = taosArrayGet(wins, i);
STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
if (!chIds) {
SPullWindowInfo pull = {
.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
@ -698,7 +708,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild);
if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,winKey, sizeof(SWinKey), NULL, 0);
tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0);
}
qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild);
}
@ -751,7 +761,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat
if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = groupId;
void* tbname = NULL;
if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, false) < 0) {
if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,
false) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -829,9 +840,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN
return startPos;
}
bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) {
return pInfo->primaryPkIndex != -1;
}
bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; }
static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId,
SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) {
@ -852,23 +861,22 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData;
void* pPkVal = NULL;
int32_t pkLen = 0;
void* pPkVal = NULL;
int32_t pkLen = 0;
SColumnInfoData* pPkColDataInfo = NULL;
if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
}
if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) {
qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64
",maxKey %" PRId64,
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64,
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex);
// timestamp of the data is incorrect
if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {
qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
}
}
@ -916,7 +924,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild);
if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0);
tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);
}
}
} else {
@ -962,7 +970,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
};
if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) {
tSimpleHashPut(pDeletedMap,&key, sizeof(SWinKey), NULL, 0);
tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0);
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
@ -1365,7 +1373,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
pInfo->binfo.pRes->info.type = pBlock->info.type;
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) {
SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL;
doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap);
if (IS_FINAL_INTERVAL_OP(pOperator)) {
@ -1399,7 +1407,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);
continue;
} else if (pBlock->info.type == STREAM_RETRIEVE) {
if(!IS_FINAL_INTERVAL_OP(pOperator)) {
if (!IS_FINAL_INTERVAL_OP(pOperator)) {
pInfo->recvRetrive = true;
copyDataBlock(pInfo->pMidRetriveRes, pBlock);
pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;
@ -1525,8 +1533,8 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info;
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&
pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
int32_t size = 0;
void* pBuf = NULL;
int32_t size = 0;
void* pBuf = NULL;
int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,
strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size);
if (code == 0) {
@ -1600,7 +1608,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pInfo->pState, &pTaskInfo->storageAPI.functionStore);
pInfo->pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1755,27 +1763,37 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
}
}
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) {
int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
pScanInfo->tsColIndex = tsColIndex;
}
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic);
return;
code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic);
return code;
}
SStreamScanInfo* pScanInfo = downstream->info;
pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
pScanInfo->pState = pAggSup->pState;
if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,
pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen);
code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,
pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen,
&pScanInfo->pUpdateInfo);
TSDB_CHECK_CODE(code, lino, _end);
}
pScanInfo->twAggSup = *pTwSup;
pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;
pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static TSKEY sesionTs(void* pKey) {
@ -2148,8 +2166,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;
void* pPkVal = NULL;
int32_t pkLen = 0;
void* pPkVal = NULL;
int32_t pkLen = 0;
SColumnInfoData* pPkColDataInfo = NULL;
if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
@ -2454,8 +2472,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa
pBlock->info.id.groupId = pKey->groupId;
void* tbname = NULL;
if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId,
&tbname, false) < 0) {
if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,
false) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -2935,7 +2953,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) {
int32_t size = 0;
void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY));
@ -2968,7 +2986,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) {
int32_t size = 0;
void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY));
@ -3018,6 +3036,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
int32_t numOfCols = 0;
int32_t code = TSDB_CODE_OUT_OF_MEMORY;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -3112,8 +3131,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
}
return pOperator;
@ -3124,6 +3147,7 @@ _error:
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}
@ -3574,7 +3598,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (!allEqual) {
uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
@ -3681,9 +3705,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
SSessionKey key = {0};
SResultWindowInfo winfo = {0};
buf = decodeSSessionKey(buf, &key);
pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL,
pAggSup->stateKeySize, compareStateKey,
(void**)&winfo.pStatePos, &pAggSup->resultRowSize);
pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize,
compareStateKey, (void**)&winfo.pStatePos,
&pAggSup->resultRowSize);
buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
}
@ -3877,7 +3901,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) {
int32_t size = 0;
void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME,
strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size);
strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
qDebug("===stream=== reload state. get result count:%d", num);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
@ -3941,12 +3965,13 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey);
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
TSDB_CHECK_CODE(code, lino, _error);
}
pInfo->stateCol = extractColumnFromColumnNode(pColNode);
@ -3955,9 +3980,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
}
pInfo->twAggSup = (STimeWindowAggSupp){
@ -3984,9 +4007,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize,
type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);
pInfo->pDelIterator = NULL;
@ -3999,7 +4021,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) {
goto _error;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
if (pHandle) {
pInfo->isHistoryOp = pHandle->fillHistory;
@ -4026,17 +4049,20 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic);
code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
TSDB_CHECK_CODE(code, lino, _error);
return pOperator;
_error:
destroyStreamStateOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL;
}
@ -4203,11 +4229,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
.precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
};
pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,
.calTrigger = pIntervalPhyNode->window.triggerType,
.maxTs = INT64_MIN,
.minTs = INT64_MAX,
.deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};
pInfo->twAggSup =
(STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,
.calTrigger = pIntervalPhyNode->window.triggerType,
.maxTs = INT64_MIN,
.minTs = INT64_MAX,
.deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};
ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
@ -4335,7 +4362,7 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
.ts = nextWin.skey,
.groupId = groupId,
};
void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));
void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));
int32_t index = -1;
SArray* chArray = NULL;
int32_t chId = 0;
@ -4407,8 +4434,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) {
int32_t size = taosArrayGetSize(wins);
for (int32_t i = 0; i < size; i++) {
SWinKey* winKey = taosArrayGet(wins, i);
void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));
SWinKey* winKey = taosArrayGet(wins, i);
void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));
if (!chIds) {
addPullWindow(pMidPullMap, winKey, numOfChild);
qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild);
@ -4611,5 +4638,6 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
void setStreamOperatorCompleted(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator);
qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo),
getStreamOpName(pOperator->operatorType), pOperator->status);
}

1
source/libs/executor/src/timewindowoperator.c
View File

@ -1139,7 +1139,6 @@ void destroyIntervalOperatorInfo(void* param) {
taosArrayDestroy(pInfo->pInterpCols);
pInfo->pInterpCols = NULL;
taosArrayDestroyEx(pInfo->pPrevValues, freeItem);
pInfo->pPrevValues = NULL;
cleanupGroupResInfo(&pInfo->groupResInfo);

101
source/libs/parser/src/parInsertSml.c
View File

@ -112,7 +112,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
for (int i = 0; i < tags->numOfBound; ++i) {
SSchema* pTagSchema = &pSchema[tags->pColIndex[i]];
SSmlKv* kv = taosArrayGet(cols, i);
if (kv == NULL){
code = TSDB_CODE_SML_INVALID_DATA;
uError("SML smlBuildTagRow error kv is null");
goto end;
}
if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 ||
kv->type != pTagSchema->type) {
code = TSDB_CODE_SML_INVALID_DATA;
@ -120,7 +124,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
goto end;
}
taosArrayPush(*tagName, pTagSchema->name);
if (taosArrayPush(*tagName, pTagSchema->name) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML smlBuildTagRow error push tag name");
goto end;
}
STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
// strcpy(val.colName, pTagSchema->name);
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY ||
@ -141,7 +149,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
goto end;
}
char buf[512] = {0};
snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
(void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
taosMemoryFree(p);
code = buildSyntaxErrMsg(msg, buf, kv->value);
goto end;
@ -149,9 +157,13 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
val.pData = p;
val.nData = output;
} else {
memcpy(&val.i64, &(kv->value), kv->length);
(void)memcpy(&val.i64, &(kv->value), kv->length);
}
if (taosArrayPush(pTagArray, &val) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML smlBuildTagRow error push tag array");
goto end;
}
taosArrayPush(pTagArray, &val);
}
code = tTagNew(pTagArray, 1, false, ppTag);
@ -166,20 +178,19 @@ end:
return code;
}
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) {
STableDataCxt* pTableCxt = NULL;
int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) {
SVCreateTbReq* pCreateTbReq = NULL;
int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false);
sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false);
if (ret != TSDB_CODE_SUCCESS) {
return NULL;
return ret;
}
ret = initTableColSubmitData(pTableCxt);
ret = initTableColSubmitData(*cxt);
if (ret != TSDB_CODE_SUCCESS) {
return NULL;
return ret;
}
return pTableCxt;
return TSDB_CODE_SUCCESS;
}
void clearColValArraySml(SArray* pCols) {
@ -197,6 +208,9 @@ void clearColValArraySml(SArray* pCols) {
int32_t smlBuildRow(STableDataCxt* pTableCxt) {
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
if (pRow == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS != ret) {
return ret;
@ -211,18 +225,22 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
int ret = TSDB_CODE_SUCCESS;
SSchema* pColSchema = schema + index;
SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
if (pVal == NULL) {
return TSDB_CODE_SUCCESS;
}
SSmlKv* kv = (SSmlKv*)data;
if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 ||
kv->type != pColSchema->type) {
ret = TSDB_CODE_SML_INVALID_DATA;
char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1);
if (tmp) {
memcpy(tmp, kv->key, kv->keyLen);
(void)memcpy(tmp, kv->key, kv->keyLen);
uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name,
pColSchema->name, tDataTypes[pColSchema->type].name);
taosMemoryFree(tmp);
} else {
uError("SML smlBuildCol out of memory");
ret = TSDB_CODE_OUT_OF_MEMORY;
}
goto end;
}
@ -256,9 +274,9 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
(void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
} else {
memcpy(&pVal->value.val, &(kv->value), kv->length);
(void)memcpy(&pVal->value.val, &(kv->value), kv->length);
}
pVal->flag = CV_FLAG_VALUE;
@ -276,8 +294,13 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
SVCreateTbReq* pCreateTblReq = NULL;
SArray* tagName = NULL;
insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags);
int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true);
int ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags);
if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "init bound cols error");
goto end;
}
ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true);
if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "bound tags error");
goto end;
@ -299,7 +322,11 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
ttl);
pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1);
memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);
if (pCreateTblReq->ctb.stbName == NULL){
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
(void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);
if (dataFormat) {
STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj,
@ -345,11 +372,18 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
for (int32_t r = 0; r < rowNum; ++r) {
void* rowData = taosArrayGetP(cols, r);
if (rowData == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
goto end;
}
// 1. set the parsed value from sql string
for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) {
SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]];
SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]);
if (pVal == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
goto end;
}
void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
if (p == NULL) {
continue;
@ -388,14 +422,22 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
if (NULL == pVal->value.pData) {
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
(void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
} else {
memcpy(&pVal->value.val, &(kv->value), kv->length);
(void)memcpy(&pVal->value.val, &(kv->value), kv->length);
}
pVal->flag = CV_FLAG_VALUE;
}
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
if (NULL == pRow) {
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS != ret) {
buildInvalidOperationMsg(&pBuf, "tRowBuild error");
@ -415,11 +457,12 @@ end:
return ret;
}
SQuery* smlInitHandle() {
int32_t smlInitHandle(SQuery** query) {
*query = NULL;
SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == pQuery) {
uError("create pQuery error");
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->haveResultSet = false;
@ -428,14 +471,22 @@ SQuery* smlInitHandle() {
if (NULL == stmt) {
uError("create SVnodeModifyOpStmt error");
qDestroyQuery(pQuery);
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (stmt->pTableBlockHashObj == NULL){
uError("create pTableBlockHashObj error");
qDestroyQuery(pQuery);
nodesDestroyNode((SNode*)stmt);
return TSDB_CODE_OUT_OF_MEMORY;
}
stmt->freeHashFunc = insDestroyTableDataCxtHashMap;
stmt->freeArrayFunc = insDestroyVgroupDataCxtList;
pQuery->pRoot = (SNode*)stmt;
return pQuery;
*query = pQuery;
return TSDB_CODE_SUCCESS;
}
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) {

6
source/libs/parser/src/parTranslater.c
View File

@ -12815,6 +12815,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt)
int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta);
uint8_t precision = getTableInfo(pParFileCxt->pStbMeta).precision;
SSchema* pSchemas = getTableTagSchema(pParFileCxt->pStbMeta);
bool isJson = false;
for (int i = 0; i < sz; i++) {
const char* pSql = pParFileCxt->pSql;
@ -12832,6 +12833,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt)
// parse tag
const SSchema* pTagSchema = &pSchemas[index];
isJson = (pTagSchema->type == TSDB_DATA_TYPE_JSON);
code = checkAndTrimValue(&token, pParFileCxt->tmpTokenBuf, pMsgBuf, pTagSchema->type);
if (TSDB_CODE_SUCCESS == code && TK_NK_VARIABLE == token.type) {
code = buildInvalidOperationMsg(pMsgBuf, "not expected row value");
@ -12859,7 +12861,9 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt)
if (TSDB_CODE_SUCCESS == code) {
pParFileCxt->tagNameFilled = true;
code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag);
if (!isJson) {
code = tTagNew(pParFileCxt->aTagVals, 1, false, &pParFileCxt->pTag);
}
}
return code;

6
source/libs/qworker/inc/qwInt.h
View File

@ -218,8 +218,8 @@ typedef struct SQWorkerMgmt {
#define QW_IDS() sId, qId, tId, rId, eId
#define QW_FPARAMS() mgmt, QW_IDS()
#define QW_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n)
#define QW_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n)
#define QW_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n)
#define QW_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n)
#define QW_STAT_GET(_item) atomic_load_64(&(_item))
#define QW_GET_EVENT(ctx, event) atomic_load_8(&(ctx)->events[event])
@ -250,7 +250,7 @@ typedef struct SQWorkerMgmt {
} while (0)
#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code)
#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_UPDATE_RSP_CODE(ctx, code) (void)atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY)
#define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch)

6
source/libs/qworker/src/qwDbg.c
View File

@ -221,11 +221,11 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) {
SEpSet epSet = {0};
epSet.inUse = 1;
epSet.numOfEps = 3;
strcpy(epSet.eps[0].fqdn, "localhost");
TAOS_STRCPY(epSet.eps[0].fqdn, "localhost");
epSet.eps[0].port = 7100;
strcpy(epSet.eps[1].fqdn, "localhost");
TAOS_STRCPY(epSet.eps[1].fqdn, "localhost");
epSet.eps[1].port = 7200;
strcpy(epSet.eps[2].fqdn, "localhost");
TAOS_STRCPY(epSet.eps[2].fqdn, "localhost");
epSet.eps[2].port = 7300;
ctx->phase = QW_PHASE_POST_QUERY;

4
source/libs/qworker/src/qwMsg.c
View File

@ -21,7 +21,7 @@ int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **r
}
if (NULL == *rsp) {
memset(pRsp, 0, sizeof(SRetrieveTableRsp));
TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp));
}
*rsp = pRsp;
@ -169,7 +169,7 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve
if (NULL == pRsp) {
QW_RET(terrno);
}
memset(pRsp, 0, sizeof(SRetrieveTableRsp));
TAOS_MEMSET(pRsp, 0, sizeof(SRetrieveTableRsp));
dataLength = 0;
}

6
source/libs/qworker/src/qwUtil.c
View File

@ -342,7 +342,7 @@ int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
if (NULL == pExec) {
QW_ERR_JRET(terrno);
}
memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo));
(void)memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo));
localRsp.rsp.subplanInfo = pExec;
localRsp.qId = qId;
localRsp.tId = tId;
@ -537,7 +537,7 @@ int32_t qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) {
}
if (dbFName[0] && tbName[0]) {
sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName);
(void)sprintf(tbInfo.tbFName, "%s.%s", dbFName, tbName);
} else {
tbInfo.tbFName[0] = 0;
}
@ -613,7 +613,7 @@ void qwDestroyImpl(void *pMgmt) {
taosMemoryFree(mgmt);
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
(void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
qwCloseRef();

20
source/libs/qworker/src/qworker.c
View File

@ -452,8 +452,8 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes
pRes->skey = pDelRes->skey;
pRes->ekey = pDelRes->ekey;
pRes->affectedRows = pDelRes->affectedRows;
strcpy(pRes->tableFName, pDelRes->tableName);
strcpy(pRes->tsColName, pDelRes->tsColName);
TAOS_STRCPY(pRes->tableFName, pDelRes->tableName);
TAOS_STRCPY(pRes->tsColName, pDelRes->tsColName);
taosMemoryFree(output.pData);
return TSDB_CODE_SUCCESS;
@ -1126,8 +1126,8 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
sch->hbConnInfo.handle = NULL;
}
memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo));
memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId));
(void)memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo));
(void)memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId));
QW_UNLOCK(QW_WRITE, &sch->hbConnLock);
@ -1138,7 +1138,7 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
_return:
memcpy(&rsp.epId, &req->epId, sizeof(req->epId));
(void)memcpy(&rsp.epId, &req->epId, sizeof(req->epId));
code = qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code);
if (code) {
@ -1299,19 +1299,19 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S
int32_t qwNum = atomic_add_fetch_32(&gQwMgmt.qwNum, 1);
if (1 == qwNum) {
memset(gQwMgmt.param, 0, sizeof(gQwMgmt.param));
TAOS_MEMSET(gQwMgmt.param, 0, sizeof(gQwMgmt.param));
}
int32_t code = qwOpenRef();
if (code) {
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
(void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
QW_RET(code);
}
SQWorker *mgmt = taosMemoryCalloc(1, sizeof(SQWorker));
if (NULL == mgmt) {
qError("calloc %d failed", (int32_t)sizeof(SQWorker));
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
(void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
QW_RET(TSDB_CODE_OUT_OF_MEMORY);
}
@ -1345,7 +1345,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S
if (pMsgCb) {
mgmt->msgCb = *pMsgCb;
} else {
memset(&mgmt->msgCb, 0, sizeof(mgmt->msgCb));
TAOS_MEMSET(&mgmt->msgCb, 0, sizeof(mgmt->msgCb));
}
mgmt->refId = taosAddRef(gQwMgmt.qwRef, mgmt);
@ -1379,7 +1379,7 @@ _return:
taosTmrCleanUp(mgmt->timer);
taosMemoryFreeClear(mgmt);
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
(void)atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
}
QW_RET(code);

4
source/libs/qworker/test/qworkerTests.cpp
View File

@ -170,7 +170,7 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) {
printf("malloc failed");
assert(0);
}
memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
(void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
qwtTestFetchQueue[qwtTestFetchQueueWIdx++] = newMsg;
if (qwtTestFetchQueueWIdx >= qwtTestFetchQueueSize) {
qwtTestFetchQueueWIdx = 0;
@ -199,7 +199,7 @@ int32_t qwtPutReqToQueue(void *node, EQueueType qtype, struct SRpcMsg *pMsg) {
printf("malloc failed");
assert(0);
}
memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
(void)memcpy(newMsg, pMsg, sizeof(struct SRpcMsg));
qwtTestQueryQueue[qwtTestQueryQueueWIdx++] = newMsg;
if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) {
qwtTestQueryQueueWIdx = 0;

21
source/libs/scheduler/inc/schInt.h
View File

@ -336,7 +336,7 @@ extern SSchedulerMgmt schMgmt;
((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \
(!SCH_IS_DATA_BIND_QRY_TASK(_task)))
#define SCH_UPDATE_REDIRECT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code)
#define SCH_UPDATE_REDIRECT_CODE(job, _code) (void)atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code)
#define SCH_GET_REDIRECT_CODE(job, _code) \
(((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode)
@ -413,7 +413,7 @@ extern SSchedulerMgmt schMgmt;
#define SCH_LOG_TASK_START_TS(_task) \
do { \
int64_t us = taosGetTimestampUs(); \
taosArrayPush((_task)->profile.execTime, &us); \
(void)taosArrayPush((_task)->profile.execTime, &us); \
if (0 == (_task)->execId) { \
(_task)->profile.startTs = us; \
} \
@ -422,7 +422,10 @@ extern SSchedulerMgmt schMgmt;
#define SCH_LOG_TASK_WAIT_TS(_task) \
do { \
int64_t us = taosGetTimestampUs(); \
(_task)->profile.waitTime += us - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId); \
int64_t* startus = (int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId); \
if (NULL != startus) { \
(_task)->profile.waitTime += us - *startus; \
} \
} while (0)
#define SCH_LOG_TASK_END_TS(_task) \
@ -430,7 +433,9 @@ extern SSchedulerMgmt schMgmt;
int64_t us = taosGetTimestampUs(); \
int32_t idx = (_task)->execId % (_task)->maxExecTimes; \
int64_t *startts = taosArrayGet((_task)->profile.execTime, (_task)->execId); \
*startts = us - *startts; \
if (NULL != startts) { \
*startts = us - *startts; \
} \
(_task)->profile.endTs = us; \
} while (0)
@ -538,7 +543,7 @@ void schCleanClusterHb(void *pTrans);
int32_t schLaunchTask(SSchJob *job, SSchTask *task);
int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask);
int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType, void *param);
SSchJob *schAcquireJob(int64_t refId);
int32_t schAcquireJob(int64_t refId, SSchJob **ppJob);
int32_t schReleaseJob(int64_t refId);
void schFreeFlowCtrl(SSchJob *pJob);
int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel);
@ -578,12 +583,12 @@ int32_t schJobFetchRows(SSchJob *pJob);
int32_t schJobFetchRowsA(SSchJob *pJob);
int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId);
int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList);
char *schDumpEpSet(SEpSet *pEpSet);
int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes);
char *schGetOpStr(SCH_OP_TYPE type);
int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync);
int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq);
int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq);
int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes);
void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes);
int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet);
int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode);
void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode);
@ -606,7 +611,7 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask);
void schDropTaskInHashList(SSchJob *pJob, SHashObj *list);
int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask);
int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level);
int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask);
void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask);
int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel);
int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask);
void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode);

12
source/libs/scheduler/src/schDbg.c
View File

@ -16,17 +16,5 @@
#include "query.h"
#include "schInt.h"
tsem_t schdRspSem;
SSchDebug gSCHDebug = {0};
void schdExecCallback(SExecResult* pResult, void* param, int32_t code) {
if (code) {
pResult->code = code;
}
*(SExecResult*)param = *pResult;
taosMemoryFree(pResult);
tsem_post(&schdRspSem);
}

13
source/libs/scheduler/src/schFlowCtrl.c
View File

@ -50,6 +50,10 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks);
for (int32_t i = 0; i < taskNum; ++i) {
SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i);
if (NULL == pTask) {
SCH_JOB_DLOG("fail to get the %dth task", i);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
sum += pTask->plan->execNodeStat.tableNum;
}
@ -214,6 +218,10 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
for (int32_t i = 0; i < taskNum; ++i) {
pTask = *(SSchTask **)taosArrayGet(ctrl->taskList, i);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task", i);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) {
@ -243,6 +251,11 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) {
if (i < (taskNum - 1)) {
SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList);
if (NULL == pLastTask) {
SCH_JOB_ELOG("fail to get the last task, num:%d", (int32_t)taosArrayGetSize(ctrl->taskList));
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (remainNum < pLastTask->plan->execNodeStat.tableNum) {
SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d, smallestInList:%d",
ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum);

164
source/libs/scheduler/src/schJob.c
View File

@ -47,6 +47,7 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) {
return;
_return:
SCH_JOB_DLOG("job errCode updated to %s", tstrerror(errCode));
}
@ -166,9 +167,18 @@ _return:
int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t i = 0; i < pJob->levelNum; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, pJob->levelNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
for (int32_t m = 0; m < pLevel->taskNum; ++m) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, m);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum: %d", m, pLevel->level, pLevel->taskNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSubplan *pPlan = pTask->plan;
int32_t childNum = pPlan->pChildren ? (int32_t)LIST_LENGTH(pPlan->pChildren) : 0;
int32_t parentNum = pPlan->pParents ? (int32_t)LIST_LENGTH(pPlan->pParents) : 0;
@ -188,6 +198,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t n = 0; n < childNum; ++n) {
SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n);
if (NULL == child) {
SCH_JOB_ELOG("fail to get the %dth child subplan, childNum: %d", n, childNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES);
if (NULL == childTask || NULL == *childTask) {
SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
@ -222,6 +237,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
for (int32_t n = 0; n < parentNum; ++n) {
SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n);
if (NULL == parent) {
SCH_JOB_ELOG("fail to get the %dth parent subplan, parentNum: %d", n, parentNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES);
if (NULL == parentTask || NULL == *parentTask) {
SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
@ -241,6 +261,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
}
SSchLevel *pLevel = taosArrayGet(pJob->levels, 0);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get level 0 level, levelNum:%d", (int32_t)taosArrayGetSize(pJob->levels));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (SCH_IS_QUERY_JOB(pJob)) {
if (pLevel->taskNum > 1) {
SCH_JOB_ELOG("invalid query plan, level:0, taskNum:%d", pLevel->taskNum);
@ -248,6 +273,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
}
SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the first task in level 0, taskNum:%d", (int32_t)taosArrayGetSize(pLevel->subTasks));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType || EXPLAIN_MODE_DISABLE != pJob->attr.explainMode) {
pJob->attr.needFetch = true;
}
@ -261,7 +291,9 @@ int32_t schAppendJobDataSrc(SSchJob *pJob, SSchTask *pTask) {
return TSDB_CODE_SUCCESS;
}
taosArrayPush(pJob->dataSrcTasks, &pTask);
if (NULL == taosArrayPush(pJob->dataSrcTasks, &pTask)) {
SCH_ERR_RET(terrno);
}
return TSDB_CODE_SUCCESS;
}
@ -318,6 +350,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
}
pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, levelNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
pLevel->level = i;
plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i);
@ -342,6 +379,10 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
for (int32_t n = 0; n < taskNum; ++n) {
SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n);
if (NULL == plan) {
SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d", n, taskNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_SET_JOB_TYPE(pJob, plan->subplanType);
@ -383,7 +424,7 @@ _return:
SCH_RET(code);
}
int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) {
void schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) {
pRes->code = atomic_load_32(&pJob->errCode);
pRes->numOfRows = pJob->resNumOfRows;
@ -395,8 +436,6 @@ int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) {
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
SCH_JOB_DLOG("execRes dumped, code: %s", tstrerror(pRes->code));
return TSDB_CODE_SUCCESS;
}
int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) {
@ -421,10 +460,13 @@ int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) {
if (NULL == *pData) {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp));
if (rsp) {
rsp->completed = 1;
if (NULL == rsp) {
SCH_JOB_ELOG("malloc SRetrieveTableRsp %d failed, code:%x", (int32_t)sizeof(SRetrieveTableRsp), terrno);
SCH_ERR_JRET(terrno);
}
rsp->completed = 1;
*pData = rsp;
SCH_JOB_DLOG("empty res and set query complete, code:%x", code);
}
@ -440,10 +482,13 @@ _return:
int32_t schNotifyUserExecRes(SSchJob *pJob) {
SExecResult *pRes = taosMemoryCalloc(1, sizeof(SExecResult));
if (pRes) {
schDumpJobExecRes(pJob, pRes);
if (NULL == pRes) {
qError("malloc execResult %d failed, error: %x", (int32_t)sizeof(SExecResult), terrno);
SCH_RET(terrno);
}
schDumpJobExecRes(pJob, pRes);
SCH_JOB_DLOG("sch start to invoke exec cb, code: %s", tstrerror(pJob->errCode));
(*pJob->userRes.execFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode));
SCH_JOB_DLOG("sch end from exec cb, code: %s", tstrerror(pJob->errCode));
@ -454,7 +499,10 @@ int32_t schNotifyUserExecRes(SSchJob *pJob) {
int32_t schNotifyUserFetchRes(SSchJob *pJob) {
void *pRes = NULL;
schDumpJobFetchRes(pJob, &pRes);
int32_t code = schDumpJobFetchRes(pJob, &pRes);
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_SUCCESS == atomic_load_32(&pJob->errCode)) {
atomic_store_32(&pJob->errCode, code);
}
SCH_JOB_DLOG("sch start to invoke fetch cb, code: %s", tstrerror(pJob->errCode));
(*pJob->userRes.fetchFp)(pRes, pJob->userRes.cbParam, atomic_load_32(&pJob->errCode));
@ -478,13 +526,13 @@ void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) {
if (SCH_JOB_IN_SYNC_OP(pJob)) {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
tsem_post(&pJob->rspSem);
(void)tsem_post(&pJob->rspSem); // ignore error
} else if (SCH_JOB_IN_ASYNC_EXEC_OP(pJob)) {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
schNotifyUserExecRes(pJob);
(void)schNotifyUserExecRes(pJob); // ignore error
} else if (SCH_JOB_IN_ASYNC_FETCH_OP(pJob)) {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
schNotifyUserFetchRes(pJob);
(void)schNotifyUserFetchRes(pJob); // ignore error
} else {
SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock);
SCH_JOB_ELOG("job not in any operation, status:%s", jobTaskStatusStr(pJob->status));
@ -519,7 +567,8 @@ int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode) {
return TSDB_CODE_SCH_IGNORE_ERROR;
}
schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode);
(void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); // ignore error
return TSDB_CODE_SCH_IGNORE_ERROR;
}
@ -530,7 +579,8 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) {
return TSDB_CODE_SCH_IGNORE_ERROR;
}
schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode);
(void)schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); // ignore error
return TSDB_CODE_SCH_IGNORE_ERROR;
}
@ -569,12 +619,21 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) {
SSchLevel *pLevel = pTask->level;
int32_t doneNum = atomic_load_32(&pLevel->taskExecDoneNum);
if (doneNum == pLevel->taskNum) {
atomic_sub_fetch_32(&pJob->levelIdx, 1);
(void)atomic_sub_fetch_32(&pJob->levelIdx, 1);
pLevel = taosArrayGet(pJob->levels, pJob->levelIdx);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
for (int32_t i = 0; i < pLevel->taskNum; ++i) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, i);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", i, pLevel->level, pLevel->taskNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
if (pTask->children && taosArrayGetSize(pTask->children) > 0) {
continue;
}
@ -602,7 +661,11 @@ int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) {
}
}
taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo));
if (NULL == taosArrayAddBatch((SArray *)pJob->execRes.res, taosArrayGet(rsp->tbVerInfo, 0), taosArrayGetSize(rsp->tbVerInfo))) {
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
SCH_ERR_RET(terrno);
}
taosArrayDestroy(rsp->tbVerInfo);
pJob->execRes.msgType = TDMT_SCH_QUERY;
@ -629,6 +692,11 @@ int32_t schLaunchJob(SSchJob *pJob) {
SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL));
} else {
SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx);
if (NULL == level) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", pJob->levelIdx, (int32_t)taosArrayGetSize(pJob->levels));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_ERR_RET(schLaunchLevelTasks(pJob, level));
}
@ -661,10 +729,19 @@ void schFreeJobImpl(void *job) {
int32_t numOfLevels = taosArrayGetSize(pJob->levels);
for (int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels);
continue;
}
int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
for (int32_t j = 0; j < numOfTasks; ++j) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth task, taskNum:%d", j, numOfTasks);
continue;
}
schFreeTask(pJob, pTask);
}
@ -687,12 +764,12 @@ void schFreeJobImpl(void *job) {
destroyQueryExecRes(&pJob->execRes);
qDestroyQueryPlan(pJob->pDag);
nodesReleaseAllocatorWeakRef(pJob->allocatorRefId);
(void)nodesReleaseAllocatorWeakRef(pJob->allocatorRefId); // ignore error
taosMemoryFreeClear(pJob->userRes.execRes);
taosMemoryFreeClear(pJob->fetchRes);
taosMemoryFreeClear(pJob->sql);
tsem_destroy(&pJob->rspSem);
(void)tsem_destroy(&pJob->rspSem); // ignore error
taosMemoryFree(pJob);
int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1);
@ -711,7 +788,7 @@ int32_t schJobFetchRows(SSchJob *pJob) {
if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) {
SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
tsem_wait(&pJob->rspSem);
(void)tsem_wait(&pJob->rspSem); // ignore error
SCH_RET(schDumpJobFetchRes(pJob, pJob->userRes.fetchRes));
}
} else {
@ -739,9 +816,19 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
pJob->conn = *pReq->pConn;
if (pReq->sql) {
pJob->sql = taosStrdup(pReq->sql);
if (NULL == pJob->sql) {
qError("QID:0x%" PRIx64 " strdup sql %s failed", pReq->pDag->queryId, pReq->sql);
SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
pJob->pDag = pReq->pDag;
pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId);
if (pReq->allocatorRefId > 0) {
pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId);
if (pJob->allocatorRefId <= 0) {
qError("QID:0x%" PRIx64 " nodesMakeAllocatorWeakRef failed", pReq->pDag->queryId);
SCH_ERR_JRET(terrno);
}
}
pJob->chkKillFp = pReq->chkKillFp;
pJob->chkKillParam = pReq->chkKillParam;
pJob->userRes.execFp = pReq->execFp;
@ -752,6 +839,10 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId);
} else {
pJob->nodeList = taosArrayDup(pReq->pNodeList, NULL);
if (NULL == pJob->nodeList) {
qError("QID:0x%" PRIx64 " taosArrayDup failed, origNum:%d", pReq->pDag->queryId, (int32_t)taosArrayGetSize(pReq->pNodeList));
SCH_ERR_JRET(terrno);
}
}
pJob->taskList = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false,
@ -785,7 +876,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
SCH_ERR_JRET(terrno);
}
atomic_add_fetch_32(&schMgmt.jobNum, 1);
(void)atomic_add_fetch_32(&schMgmt.jobNum, 1);
*pJobId = pJob->refId;
@ -800,7 +891,7 @@ _return:
} else if (pJob->refId < 0) {
schFreeJobImpl(pJob);
} else {
taosRemoveRef(schMgmt.jobRef, pJob->refId);
(void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error
}
SCH_RET(code);
@ -814,7 +905,7 @@ int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq) {
if (pReq->syncReq) {
SCH_JOB_DLOG("sync wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
tsem_wait(&pJob->rspSem);
(void)tsem_wait(&pJob->rspSem); // ignore error
}
SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId);
@ -845,7 +936,7 @@ int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) {
}
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC);
SCH_ERR_RET(schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC));
}
return TSDB_CODE_SUCCESS;
@ -866,6 +957,10 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) {
int32_t numOfLevels = taosArrayGetSize(pJob->levels);
for (int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
SCH_JOB_ELOG("fail to get the %dth level, levelNum:%d", i, numOfLevels);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
pLevel->taskExecDoneNum = 0;
pLevel->taskLaunchedNum = 0;
@ -873,6 +968,11 @@ int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode, bool *inRetry) {
int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
for (int32_t j = 0; j < numOfTasks; ++j) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
if (NULL == pTask) {
SCH_JOB_ELOG("fail to get the %dth task in level %d, taskNum:%d", j, i, numOfTasks);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_LOCK_TASK(pTask);
code = schChkUpdateRedirectCtx(pJob, pTask, NULL, rspCode);
if (TSDB_CODE_SUCCESS != code) {
@ -973,7 +1073,7 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int
}
if (errCode) {
schHandleJobFailure(pJob, errCode);
(void)schHandleJobFailure(pJob, errCode); // handle internal
}
SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode));
@ -1052,11 +1152,11 @@ void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode) {
}
if (errCode) {
schHandleJobFailure(pJob, errCode);
(void)schHandleJobFailure(pJob, errCode); // ignore error
}
if (pJob) {
schReleaseJob(pJob->refId);
(void)schReleaseJob(pJob->refId); // ignore error
}
}
@ -1065,7 +1165,9 @@ int32_t schProcessOnCbBegin(SSchJob **job, SSchTask **task, uint64_t qId, int64_
int8_t status = 0;
SSchTask *pTask = NULL;
SSchJob *pJob = schAcquireJob(rId);
SSchJob *pJob = NULL;
(void)schAcquireJob(rId, &pJob);
if (NULL == pJob) {
qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, qId, tId, rId);
SCH_ERR_RET(TSDB_CODE_QRY_JOB_NOT_EXIST);
@ -1091,7 +1193,7 @@ _return:
SCH_UNLOCK_TASK(pTask);
}
if (pJob) {
schReleaseJob(rId);
(void)schReleaseJob(rId); // ignore error
}
SCH_RET(code);

91
source/libs/scheduler/src/schRemote.c
View File

@ -113,7 +113,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs
}
atomic_store_ptr(&pJob->fetchRes, rsp);
atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows));
(void)atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows));
if (rsp->completed) {
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC);
@ -166,13 +166,27 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD
SCH_LOCK(SCH_WRITE, &pJob->resLock);
if (NULL == pJob->execRes.res) {
pJob->execRes.res = (void*)taosArrayInit(batchRsp.nRsps, POINTER_BYTES);
if (NULL == pJob->execRes.res) {
code = terrno;
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
tDecoderClear(&coder);
SCH_ERR_JRET(code);
}
pJob->execRes.msgType = TDMT_VND_CREATE_TABLE;
}
for (int32_t i = 0; i < batchRsp.nRsps; ++i) {
SVCreateTbRsp *rsp = batchRsp.pRsps + i;
if (rsp->pMeta) {
taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta);
if (NULL == taosArrayPush((SArray*)pJob->execRes.res, &rsp->pMeta)) {
code = terrno;
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
tDecoderClear(&coder);
SCH_ERR_JRET(code);
}
}
if (TSDB_CODE_SUCCESS != rsp->code) {
@ -264,7 +278,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD
SCH_ERR_JRET(code);
}
atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows);
(void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows);
int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp);
SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum);
@ -275,7 +289,12 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD
SSubmitRsp2 *sum = pJob->execRes.res;
sum->affectedRows += rsp->affectedRows;
if (sum->aCreateTbRsp) {
taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp);
if (NULL == taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp)) {
code = terrno;
SCH_UNLOCK(SCH_WRITE, &pJob->resLock);
SCH_ERR_JRET(code);
}
taosArrayDestroy(rsp->aCreateTbRsp);
} else {
TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp);
@ -313,10 +332,14 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD
SDecoder coder = {0};
SVDeleteRsp rsp = {0};
tDecoderInit(&coder, msg, msgSize);
tDecodeSVDeleteRsp(&coder, &rsp);
if (tDecodeSVDeleteRsp(&coder, &rsp) < 0) {
code = terrno;
tDecoderClear(&coder);
SCH_ERR_JRET(code);
}
tDecoderClear(&coder);
atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows);
(void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows);
SCH_TASK_DLOG("delete succeed, affectedRows:%" PRId64, rsp.affectedRows);
}
@ -351,7 +374,7 @@ int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SD
SCH_ERR_JRET(schSaveJobExecRes(pJob, &rsp));
atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows);
(void)atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows);
taosMemoryFreeClear(msg);
@ -479,7 +502,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) {
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId,
code);
// called if drop task rsp received code
rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT);
(void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error
if (pMsg) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
@ -501,7 +524,7 @@ int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) {
int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) {
SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param;
rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT);
(void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error
qDebug("handle %p is broken", pMsg->handle);
@ -531,7 +554,7 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) {
if (code) {
qError("hb rsp error:%s", tstrerror(code));
rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT);
(void)rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); // ignore error
SCH_ERR_JRET(code);
}
@ -585,9 +608,14 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo
param->head.isHbParam = true;
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
if (NULL == addr) {
taosMemoryFree(param);
SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
param->nodeEpId.nodeId = addr->nodeId;
SEp *pEp = SCH_GET_CUR_EP(addr);
strcpy(param->nodeEpId.ep.fqdn, pEp->fqdn);
TAOS_STRCPY(param->nodeEpId.ep.fqdn, pEp->fqdn);
param->nodeEpId.ep.port = pEp->port;
param->pTrans = trans->pTrans;
*pParam = param;
@ -712,7 +740,7 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) {
int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) {
int32_t code = 0;
memcpy(pDst, pSrc, sizeof(SRpcCtx));
TAOS_MEMCPY(pDst, pSrc, sizeof(SRpcCtx));
pDst->brokenVal.val = NULL;
pDst->args = NULL;
@ -760,7 +788,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) {
SQueryNodeEpId epId = {0};
epId.nodeId = addr->nodeId;
memcpy(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp));
TAOS_MEMCPY(&epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp));
pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
if (NULL == pCtx->args) {
@ -877,7 +905,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(dst, pSrc, sizeof(*dst));
TAOS_MEMCPY(dst, pSrc, sizeof(*dst));
*pDst = (SSchCallbackParamHeader *)dst;
return TSDB_CODE_SUCCESS;
@ -889,7 +917,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(dst, pSrc, sizeof(*dst));
TAOS_MEMCPY(dst, pSrc, sizeof(*dst));
*pDst = (SSchCallbackParamHeader *)dst;
return TSDB_CODE_SUCCESS;
@ -904,7 +932,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) {
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(pDst, pSrc, sizeof(*pSrc));
TAOS_MEMCPY(pDst, pSrc, sizeof(*pSrc));
pDst->param = NULL;
SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param));
@ -948,6 +976,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery
if (isHb && persistHandle && trans->pHandle == 0) {
trans->pHandle = rpcAllocHandle();
if (NULL == trans->pHandle) {
SCH_TASK_ELOG("rpcAllocHandle failed, code:%x", terrno);
SCH_ERR_JRET(terrno);
}
}
if (pJob && pTask) {
@ -1000,7 +1032,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) {
req.header.vgId = nodeEpId->nodeId;
req.sId = schMgmt.sId;
memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId));
TAOS_MEMCPY(&req.epId, nodeEpId, sizeof(SQueryNodeEpId));
SCH_LOCK(SCH_READ, &schMgmt.hbLock);
SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId));
@ -1013,7 +1045,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) {
SCH_LOCK(SCH_WRITE, &hb->lock);
code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx);
memcpy(&trans, &hb->trans, sizeof(trans));
TAOS_MEMCPY(&trans, &hb->trans, sizeof(trans));
SCH_UNLOCK(SCH_WRITE, &hb->lock);
SCH_UNLOCK(SCH_READ, &schMgmt.hbLock);
@ -1039,7 +1071,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) {
SQueryNodeAddr addr = {.nodeId = nodeEpId->nodeId};
addr.epSet.inUse = 0;
addr.epSet.numOfEps = 1;
memcpy(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep));
TAOS_MEMCPY(&addr.epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep));
code = schAsyncSendMsg(NULL, NULL, &trans, &addr, msgType, msg, msgSize, true, &rpcCtx);
msg = NULL;
@ -1064,6 +1096,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
if (NULL == addr) {
addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
if (NULL == addr) {
SCH_TASK_ELOG("fail to get condidateAddr, candidateIdx %d, totalNum: %d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs));
SCH_ERR_JRET(terrno);
}
isCandidateAddr = true;
SCH_TASK_DLOG("target candidateIdx %d, epInUse %d/%d", pTask->candidateIdx, addr->epSet.inUse,
addr->epSet.numOfEps);
@ -1082,7 +1119,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(msg, pTask->msg, msgSize);
TAOS_MEMCPY(msg, pTask->msg, msgSize);
break;
}
@ -1098,13 +1135,21 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
req.msg = pTask->msg;
req.source = pJob->source;
msgSize = tSerializeSVDeleteReq(NULL, 0, &req);
if (msgSize < 0) {
SCH_TASK_ELOG("tSerializeSVDeleteReq failed, code:%x", terrno);
SCH_ERR_JRET(terrno);
}
msg = taosMemoryCalloc(1, msgSize);
if (NULL == msg) {
SCH_TASK_ELOG("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
tSerializeSVDeleteReq(msg, msgSize, &req);
msgSize = tSerializeSVDeleteReq(msg, msgSize, &req);
if (msgSize < 0) {
SCH_TASK_ELOG("tSerializeSVDeleteReq second failed, code:%x", terrno);
SCH_ERR_JRET(terrno);
}
break;
}
case TDMT_SCH_QUERY:
@ -1221,7 +1266,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
req.sId = schMgmt.sId;
req.header.vgId = addr->nodeId;
req.epId.nodeId = addr->nodeId;
memcpy(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp));
TAOS_MEMCPY(&req.epId.ep, SCH_GET_CUR_EP(addr), sizeof(SEp));
msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req);
if (msgSize < 0) {

7
source/libs/scheduler/src/schStatus.c
View File

@ -43,7 +43,7 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) {
SCH_RET(schProcessOnJobFailure(pJob, (param ? *(int32_t*)param : 0)));
break;
case JOB_TASK_STATUS_DROP:
schProcessOnJobDropped(pJob, *(int32_t*)param);
(void)schProcessOnJobDropped(pJob, *(int32_t*)param); // ignore error
if (taosRemoveRef(schMgmt.jobRef, pJob->refId)) {
SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, pJob->refId);
@ -65,7 +65,8 @@ _return:
}
int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) {
SSchJob* pJob = schAcquireJob(jobId);
SSchJob* pJob = NULL;
(void)schAcquireJob(jobId, &pJob);
if (NULL == pJob) {
qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId);
SCH_ERR_RET(TSDB_CODE_SCH_JOB_NOT_EXISTS);
@ -90,7 +91,7 @@ int32_t schHandleOpEndEvent(SSchJob* pJob, SCH_OP_TYPE type, SSchedulerReq* pReq
code = pJob->errCode;
}
schReleaseJob(pJob->refId);
(void)schReleaseJob(pJob->refId); // ignore error
return code;
}

124
source/libs/scheduler/src/schTask.c
View File

@ -165,7 +165,7 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v
SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId));
}
schUpdateTaskExecNode(pJob, pTask, handle, execId);
SCH_ERR_RET(schUpdateTaskExecNode(pJob, pTask, handle, execId));
if ((execId != pTask->execId || execId <= pTask->failedExecId) || pTask->waitRetry) { // ignore it
SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId,
@ -297,7 +297,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
for (int32_t i = 0; i < parentNum; ++i) {
SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i);
if (NULL == parent) {
SCH_TASK_ELOG("fail to get task %d parent, parentNum: %d", i, parentNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_LOCK(SCH_WRITE, &parent->planLock);
SDownstreamSourceNode source = {
.type = QUERY_NODE_DOWNSTREAM_SOURCE,
@ -308,9 +312,14 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
.fetchMsgType = SCH_FETCH_TYPE(pTask),
.localExec = SCH_IS_LOCAL_EXEC_TASK(pJob, pTask),
};
qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source);
code = qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source);
if (TSDB_CODE_SUCCESS != code) {
SCH_TASK_ELOG("qSetSubplanExecutionNode failed, groupId: %d", pTask->plan->id.groupId);
}
SCH_UNLOCK(SCH_WRITE, &parent->planLock);
SCH_ERR_RET(code);
int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1);
if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) {
@ -413,16 +422,16 @@ void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) {
schDropTaskOnExecNode(pJob, pTask);
if (pTask->delayTimer) {
taosTmrStopA(&pTask->delayTimer);
(void)taosTmrStopA(&pTask->delayTimer); // ignore error
}
taosHashClear(pTask->execNodes);
schRemoveTaskFromExecList(pJob, pTask);
(void)schRemoveTaskFromExecList(pJob, pTask); // ignore error
schDeregisterTaskHb(pJob, pTask);
taosMemoryFreeClear(pTask->msg);
pTask->msgLen = 0;
pTask->lastMsgType = 0;
pTask->childReady = 0;
memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr));
TAOS_MEMSET(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr));
}
int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) {
@ -443,11 +452,21 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet));
} else if (SYNC_SELF_LEADER_REDIRECT_ERROR(rspCode)) {
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
if (NULL == addr) {
SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs));
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SEp *pEp = &addr->epSet.eps[addr->epSet.inUse];
SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d, code:%s", addr->nodeId, addr->epSet.inUse,
addr->epSet.numOfEps, pEp->fqdn, pEp->port, tstrerror(rspCode));
} else {
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
if (NULL == addr) {
SCH_TASK_ELOG("fail to get the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs));
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_SWITCH_EPSET(addr);
SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps);
}
@ -476,7 +495,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
for (int32_t i = 0; i < childrenNum; ++i) {
SSchTask *pChild = taosArrayGetP(pTask->children, i);
SCH_LOCK_TASK(pChild);
schDoTaskRedirect(pJob, pChild, NULL, rspCode);
(void)schDoTaskRedirect(pJob, pChild, NULL, rspCode); // error handled internal
SCH_UNLOCK_TASK(pChild);
}
@ -494,18 +513,23 @@ int32_t schResetTaskSetLevelInfo(SSchJob *pJob, SSchTask *pTask) {
atomic_load_32(&pLevel->taskExecDoneNum), atomic_load_32(&pLevel->taskLaunchedNum));
if (SCH_GET_TASK_STATUS(pTask) >= JOB_TASK_STATUS_PART_SUCC) {
atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1);
(void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1);
}
atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1);
(void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1);
int32_t childrenNum = taosArrayGetSize(pTask->children);
for (int32_t i = 0; i < childrenNum; ++i) {
SSchTask *pChild = taosArrayGetP(pTask->children, i);
if (NULL == pChild) {
SCH_TASK_ELOG("fail to get the %dth child, childrenNum:%d", i, childrenNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_LOCK_TASK(pChild);
pLevel = pChild->level;
atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1);
atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1);
(void)atomic_sub_fetch_32(&pLevel->taskExecDoneNum, 1);
(void)atomic_sub_fetch_32(&pLevel->taskLaunchedNum, 1);
SCH_UNLOCK_TASK(pChild);
}
@ -711,9 +735,9 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
}
int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) {
atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
(void)atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
schRemoveTaskFromExecList(pJob, pTask);
(void)schRemoveTaskFromExecList(pJob, pTask); // ignore error
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) {
@ -724,6 +748,11 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) {
if (SCH_IS_DATA_BIND_TASK(pTask)) {
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
if (NULL == addr) {
SCH_TASK_ELOG("fail to the %dth condidateAddr, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SCH_SWITCH_EPSET(addr);
} else {
SCH_ERR_RET(schSwitchTaskCandidateAddr(pJob, pTask));
@ -743,6 +772,11 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) {
for (int32_t i = 0; i < nodeNum; ++i) {
SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i);
if (NULL == nload) {
SCH_TASK_ELOG("fail to get the %dth node in nodeList, nodeNum:%d", i, nodeNum);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SQueryNodeAddr *naddr = &nload->addr;
if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) {
@ -810,6 +844,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
}
int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pTask->candidateAddrs || 1 != taosArrayGetSize(pTask->candidateAddrs)) {
SCH_TASK_ELOG("not able to update cndidate addr, addr num %d",
(int32_t)(pTask->candidateAddrs ? taosArrayGetSize(pTask->candidateAddrs) : 0));
@ -817,18 +852,27 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe
}
SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0);
if (NULL == pAddr) {
SCH_TASK_ELOG("fail to get task 0th condidataAddr, totalNum:%d", (int32_t)taosArrayGetSize(pTask->candidateAddrs));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
char *origEpset = schDumpEpSet(&pAddr->epSet);
char *newEpset = schDumpEpSet(pEpSet);
char *origEpset = NULL;
char *newEpset = NULL;
SCH_ERR_RET(schDumpEpSet(&pAddr->epSet, &origEpset));
SCH_ERR_JRET(schDumpEpSet(pEpSet, &newEpset));
SCH_TASK_DLOG("update task target node %d epset from %s to %s", pAddr->nodeId, origEpset, newEpset);
TAOS_MEMCPY(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet));
_return:
taosMemoryFree(origEpset);
taosMemoryFree(newEpset);
memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet));
return TSDB_CODE_SUCCESS;
return code;
}
int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) {
@ -877,7 +921,6 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) {
}
int32_t size = (int32_t)taosHashGetSize(pTask->execNodes);
if (size <= 0) {
SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
return;
@ -889,7 +932,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) {
if (nodeInfo->handle) {
SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle);
void *pExecId = taosHashGetKey(nodeInfo, NULL);
schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId);
(void)schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, pExecId); // ignore error and continue
SCH_TASK_DLOG("start to drop task's %dth execNode", i);
} else {
@ -939,6 +982,11 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) {
for (int32_t i = 0; i < taskNum; ++i) {
STaskStatus *pStatus = taosArrayGet(pStatusList, i);
if (NULL == pStatus) {
qError("fail to get the %dth task status in hb rsp, taskNum:%d", i, taskNum);
continue;
}
int32_t code = 0;
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d task status in server: %s", pStatus->queryId, pStatus->taskId,
@ -983,10 +1031,15 @@ int32_t schHandleExplainRes(SArray *pExplainRes) {
for (int32_t i = 0; i < resNum; ++i) {
SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i);
if (NULL == localRsp) {
qError("fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum);
continue;
}
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", begin to handle LOCAL explain rsp msg", localRsp->qId, localRsp->tId);
pJob = schAcquireJob(localRsp->rId);
pJob = NULL;
(void)schAcquireJob(localRsp->rId, &pJob);
if (NULL == pJob) {
qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "job no exist, may be dropped, refId:0x%" PRIx64, localRsp->qId,
localRsp->tId, localRsp->rId);
@ -996,7 +1049,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) {
int8_t status = 0;
if (schJobNeedToStop(pJob, &status)) {
SCH_TASK_DLOG("will not do further processing cause of job status %s", jobTaskStatusStr(status));
schReleaseJob(pJob->refId);
(void)schReleaseJob(pJob->refId);
SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR);
}
@ -1006,7 +1059,7 @@ int32_t schHandleExplainRes(SArray *pExplainRes) {
code = schProcessExplainRsp(pJob, pTask, &localRsp->rsp);
}
schReleaseJob(pJob->refId);
(void)schReleaseJob(pJob->refId);
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ", end to handle LOCAL explain rsp msg, code:%x", localRsp->qId,
localRsp->tId, code);
@ -1022,6 +1075,11 @@ _return:
for (int32_t i = 0; i < resNum; ++i) {
SExplainLocalRsp *localRsp = taosArrayGet(pExplainRes, i);
if (NULL == localRsp) {
qError("in _return fail to get the %dth LOCAL explain rsp msg, total:%d", i, resNum);
continue;
}
tFreeSExplainRsp(&localRsp->rsp);
}
@ -1076,6 +1134,9 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) {
if (SCH_IS_EXPLAIN_JOB(pJob)) {
explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp));
if (NULL == explainRes) {
SCH_ERR_RET(terrno);
}
}
SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId,
@ -1097,7 +1158,9 @@ _return:
int32_t schLaunchTaskImpl(void *param) {
SSchTaskCtx *pCtx = (SSchTaskCtx *)param;
SSchJob *pJob = schAcquireJob(pCtx->jobRid);
SSchJob *pJob = NULL;
(void)schAcquireJob(pCtx->jobRid, &pJob);
if (NULL == pJob) {
qDebug("job refId 0x%" PRIx64 " already not exist", pCtx->jobRid);
taosMemoryFree(param);
@ -1113,7 +1176,7 @@ int32_t schLaunchTaskImpl(void *param) {
int8_t status = 0;
int32_t code = 0;
atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1);
(void)atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1);
pTask->execId++;
pTask->retryTimes++;
pTask->waitRetry = false;
@ -1160,7 +1223,7 @@ _return:
SCH_UNLOCK_TASK(pTask);
}
schReleaseJob(pJob->refId);
(void)schReleaseJob(pJob->refId);
taosMemoryFree(param);
@ -1178,7 +1241,7 @@ int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) {
param->asyncLaunch = true;
taosAsyncExec(schLaunchTaskImpl, param, NULL);
SCH_ERR_RET(taosAsyncExec(schLaunchTaskImpl, param, NULL));
} else {
SCH_ERR_RET(schLaunchTaskImpl(param));
}
@ -1252,7 +1315,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) {
return TSDB_CODE_SUCCESS;
}
taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer);
(void)taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer);
return TSDB_CODE_SUCCESS;
}
@ -1283,7 +1346,7 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) {
SCH_LOCK_TASK(pTask);
if (pTask->delayTimer) {
taosTmrStopA(&pTask->delayTimer);
(void)taosTmrStopA(&pTask->delayTimer);
}
schDropTaskOnExecNode(pJob, pTask);
SCH_UNLOCK_TASK(pTask);
@ -1327,6 +1390,9 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) {
if (SCH_IS_EXPLAIN_JOB(pJob)) {
explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp));
if (NULL == explainRes) {
SCH_ERR_RET(terrno);
}
}
SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId,

57
source/libs/scheduler/src/schUtil.c
View File

@ -22,9 +22,14 @@
#include "tref.h"
#include "trpc.h"
FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) {
FORCE_INLINE int32_t schAcquireJob(int64_t refId, SSchJob** ppJob) {
qDebug("sch acquire jobId:0x%" PRIx64, refId);
return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId);
*ppJob = (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId);
if (NULL == *ppJob) {
return terrno;
}
return TSDB_CODE_SUCCESS;
}
FORCE_INLINE int32_t schReleaseJob(int64_t refId) {
@ -36,15 +41,16 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) {
return taosReleaseRef(schMgmt.jobRef, refId);
}
char *schDumpEpSet(SEpSet *pEpSet) {
int32_t schDumpEpSet(SEpSet *pEpSet, char** ppRes) {
*ppRes = NULL;
if (NULL == pEpSet) {
return NULL;
return TSDB_CODE_SUCCESS;
}
int32_t maxSize = 1024;
char *str = taosMemoryMalloc(maxSize);
if (NULL == str) {
return NULL;
return terrno;
}
int32_t n = 0;
@ -54,7 +60,8 @@ char *schDumpEpSet(SEpSet *pEpSet) {
n += snprintf(str + n, maxSize - n, "[%s:%d]", pEp->fqdn, pEp->port);
}
return str;
*ppRes = str;
return TSDB_CODE_SUCCESS;
}
char *schGetOpStr(SCH_OP_TYPE type) {
@ -73,7 +80,7 @@ char *schGetOpStr(SCH_OP_TYPE type) {
}
void schFreeHbTrans(SSchHbTrans *pTrans) {
rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT);
(void)rpcReleaseHandle((void *)pTrans->trans.pHandleId, TAOS_CONN_CLIENT);
schFreeRpcCtx(&pTrans->rpcCtx);
}
@ -86,7 +93,7 @@ void schCleanClusterHb(void *pTrans) {
if (hb->trans.pTrans == pTrans) {
SQueryNodeEpId *pEpId = taosHashGetKey(hb, NULL);
schFreeHbTrans(hb);
taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId));
(void)taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId));
}
hb = taosHashIterate(schMgmt.hbConnections, hb);
@ -109,7 +116,7 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep
int64_t taskNum = atomic_load_64(&hb->taskNum);
if (taskNum <= 0) {
schFreeHbTrans(hb);
taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId));
(void)taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId));
}
SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock);
@ -165,7 +172,7 @@ int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *
break;
}
atomic_add_fetch_64(&hb->taskNum, 1);
(void)atomic_add_fetch_64(&hb->taskNum, 1);
SCH_UNLOCK(SCH_READ, &schMgmt.hbLock);
@ -178,12 +185,17 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) {
}
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
if (NULL == addr) {
SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs));
return;
}
SQueryNodeEpId epId = {0};
epId.nodeId = addr->nodeId;
SEp *pEp = SCH_GET_CUR_EP(addr);
strcpy(epId.ep.fqdn, pEp->fqdn);
TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn);
epId.ep.port = pEp->port;
SCH_LOCK(SCH_READ, &schMgmt.hbLock);
@ -197,7 +209,7 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) {
int64_t taskNum = atomic_sub_fetch_64(&hb->taskNum, 1);
if (0 == taskNum) {
SCH_UNLOCK(SCH_READ, &schMgmt.hbLock);
schRemoveHbConnection(pJob, pTask, &epId);
(void)schRemoveHbConnection(pJob, pTask, &epId);
} else {
SCH_UNLOCK(SCH_READ, &schMgmt.hbLock);
}
@ -211,12 +223,17 @@ int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) {
}
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx);
if (NULL == addr) {
SCH_TASK_ELOG("fail to get the %dth condidateAddr in task, totalNum:%d", pTask->candidateIdx, (int32_t)taosArrayGetSize(pTask->candidateAddrs));
return TSDB_CODE_SCH_INTERNAL_ERROR;
}
SQueryNodeEpId epId = {0};
epId.nodeId = addr->nodeId;
SEp *pEp = SCH_GET_CUR_EP(addr);
strcpy(epId.ep.fqdn, pEp->fqdn);
TAOS_STRCPY(epId.ep.fqdn, pEp->fqdn);
epId.ep.port = pEp->port;
SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId));
@ -240,7 +257,7 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) {
}
SCH_LOCK(SCH_WRITE, &hb->lock);
memcpy(&hb->trans, trans, sizeof(*trans));
TAOS_MEMCPY(&hb->trans, trans, sizeof(*trans));
SCH_UNLOCK(SCH_WRITE, &hb->lock);
SCH_UNLOCK(SCH_READ, &schMgmt.hbLock);
@ -256,7 +273,7 @@ void schCloseJobRef(void) {
}
if (schMgmt.jobRef >= 0) {
taosCloseRef(schMgmt.jobRef);
(void)taosCloseRef(schMgmt.jobRef);
schMgmt.jobRef = -1;
}
}
@ -316,18 +333,18 @@ void schFreeRpcCtx(SRpcCtx *pCtx) {
}
}
int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) {
void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) {
*pTask = NULL;
int32_t s = taosHashGetSize(pTaskList);
if (s <= 0) {
return TSDB_CODE_SUCCESS;
return;
}
SSchTask **task = taosHashGet(pTaskList, &taskId, sizeof(taskId));
if (NULL == task || NULL == (*task)) {
return TSDB_CODE_SUCCESS;
return;
}
*pTask = *task;
return TSDB_CODE_SUCCESS;
}

29
source/libs/scheduler/src/scheduler.c
View File

@ -108,14 +108,26 @@ int32_t schedulerGetTasksStatus(int64_t jobId, SArray *pSub) {
for (int32_t i = pJob->levelNum - 1; i >= 0; --i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
if (NULL == pLevel) {
qError("failed to get level %d", i);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
for (int32_t m = 0; m < pLevel->taskNum; ++m) {
SSchTask *pTask = taosArrayGet(pLevel->subTasks, m);
if (NULL == pTask) {
qError("failed to get task %d, total: %d", m, pLevel->taskNum);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SQuerySubDesc subDesc = {0};
subDesc.tid = pTask->taskId;
strcpy(subDesc.status, jobTaskStatusStr(pTask->status));
TAOS_STRCPY(subDesc.status, jobTaskStatusStr(pTask->status));
taosArrayPush(pSub, &subDesc);
if (NULL == taosArrayPush(pSub, &subDesc)) {
qError("taosArrayPush task %d failed, error: %x, ", m, terrno);
SCH_ERR_JRET(terrno);
}
}
}
@ -141,7 +153,7 @@ int32_t schedulerUpdatePolicy(int32_t policy) {
qDebug("schedule policy updated to %d", schMgmt.cfg.schPolicy);
break;
default:
return TSDB_CODE_TSC_INVALID_INPUT;
SCH_RET(TSDB_CODE_TSC_INVALID_INPUT);
}
return TSDB_CODE_SUCCESS;
@ -157,16 +169,17 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) {
return;
}
SSchJob *pJob = schAcquireJob(*jobId);
SSchJob *pJob = NULL;
(void)schAcquireJob(*jobId, &pJob);
if (NULL == pJob) {
qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId);
qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId);
return;
}
SCH_JOB_DLOG("start to free job 0x%" PRIx64 ", code:%s", *jobId, tstrerror(errCode));
schHandleJobDrop(pJob, errCode);
(void)schHandleJobDrop(pJob, errCode); // ignore any error
schReleaseJob(*jobId);
(void)schReleaseJob(*jobId); // ignore error
*jobId = 0;
}
@ -182,7 +195,7 @@ void schedulerDestroy(void) {
if (refId == 0) {
break;
}
taosRemoveRef(schMgmt.jobRef, pJob->refId);
(void)taosRemoveRef(schMgmt.jobRef, pJob->refId); // ignore error
pJob = taosIterateRef(schMgmt.jobRef, refId);
}

222
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -75,9 +75,7 @@ int32_t schtStartFetch = 0;
void schtInitLogFile() {
const char *defaultLogFileNamePrefix = "taoslog";
const int32_t maxLogFileNum = 10;
rpcInit();
tsAsyncLog = 0;
rpcInit();
qDebugFlag = 159;
strcpy(tsLogDir, TD_LOG_DIR_PATH);
@ -136,8 +134,13 @@ int32_t schtBuildSubmitRspMsg(uint32_t *msize, void **rspMsg) {
tEncodeSize(tEncodeSSubmitRsp2, &submitRsp, msgSize, ret);
void *msg = taosMemoryCalloc(1, msgSize);
if (NULL == msg) {
return terrno;
}
tEncoderInit(&ec, (uint8_t *)msg, msgSize);
tEncodeSSubmitRsp2(&ec, &submitRsp);
if (tEncodeSSubmitRsp2(&ec, &submitRsp) < 0) {
return -1;
}
tEncoderClear(&ec);
*rspMsg = msg;
@ -152,11 +155,26 @@ void schtBuildQueryDag(SQueryPlan *dag) {
dag->queryId = qId;
dag->numOfSubplans = 2;
dag->pSubplans = nodesMakeList();
if (NULL == dag->pSubplans) {
return;
}
SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
if (NULL == scan) {
return;
}
SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
if (NULL == merge) {
return;
}
SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
if (NULL == scanPlan) {
return;
}
SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
if (NULL == mergePlan) {
return;
}
scanPlan->id.queryId = qId;
scanPlan->id.groupId = 0x0000000000000002;
@ -170,7 +188,13 @@ void schtBuildQueryDag(SQueryPlan *dag) {
scanPlan->pChildren = NULL;
scanPlan->level = 1;
scanPlan->pParents = nodesMakeList();
if (NULL == scanPlan->pParents) {
return;
}
scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
if (NULL == scanPlan->pNode) {
return;
}
scanPlan->msgType = TDMT_SCH_QUERY;
mergePlan->id.queryId = qId;
@ -181,21 +205,33 @@ void schtBuildQueryDag(SQueryPlan *dag) {
mergePlan->execNode.epSet.numOfEps = 0;
mergePlan->pChildren = nodesMakeList();
if (NULL == mergePlan->pChildren) {
return;
}
mergePlan->pParents = NULL;
mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE);
if (NULL == mergePlan->pNode) {
return;
}
mergePlan->msgType = TDMT_SCH_QUERY;
merge->pNodeList = nodesMakeList();
if (NULL == merge->pNodeList) {
return;
}
scan->pNodeList = nodesMakeList();
if (NULL == scan->pNodeList) {
return;
}
nodesListAppend(merge->pNodeList, (SNode *)mergePlan);
nodesListAppend(scan->pNodeList, (SNode *)scanPlan);
(void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan);
(void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan);
nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan);
nodesListAppend(scanPlan->pParents, (SNode *)mergePlan);
(void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan);
(void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan);
nodesListAppend(dag->pSubplans, (SNode *)merge);
nodesListAppend(dag->pSubplans, (SNode *)scan);
(void)nodesListAppend(dag->pSubplans, (SNode *)merge);
(void)nodesListAppend(dag->pSubplans, (SNode *)scan);
}
void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
@ -205,18 +241,42 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
dag->queryId = qId;
dag->numOfSubplans = 2;
dag->pSubplans = nodesMakeList();
if (NULL == dag->pSubplans) {
return;
}
SNodeListNode *scan = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
if (NULL == scan) {
return;
}
SNodeListNode *merge = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
if (NULL == merge) {
return;
}
SSubplan *mergePlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
if (NULL == mergePlan) {
return;
}
merge->pNodeList = nodesMakeList();
if (NULL == merge->pNodeList) {
return;
}
scan->pNodeList = nodesMakeList();
if (NULL == scan->pNodeList) {
return;
}
mergePlan->pChildren = nodesMakeList();
if (NULL == mergePlan->pChildren) {
return;
}
for (int32_t i = 0; i < scanPlanNum; ++i) {
SSubplan *scanPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
if (NULL == scanPlan) {
return;
}
scanPlan->id.queryId = qId;
scanPlan->id.groupId = 0x0000000000000002;
scanPlan->id.subplanId = 0x0000000000000003 + i;
@ -233,13 +293,19 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
scanPlan->pChildren = NULL;
scanPlan->level = 1;
scanPlan->pParents = nodesMakeList();
if (NULL == scanPlan->pParents) {
return;
}
scanPlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
if (NULL == scanPlan->pNode) {
return;
}
scanPlan->msgType = TDMT_SCH_QUERY;
nodesListAppend(scanPlan->pParents, (SNode *)mergePlan);
nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan);
(void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan);
(void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan);
nodesListAppend(scan->pNodeList, (SNode *)scanPlan);
(void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan);
}
mergePlan->id.queryId = qId;
@ -251,12 +317,15 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) {
mergePlan->pParents = NULL;
mergePlan->pNode = (SPhysiNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE);
if (NULL == mergePlan->pNode) {
return;
}
mergePlan->msgType = TDMT_SCH_QUERY;
nodesListAppend(merge->pNodeList, (SNode *)mergePlan);
(void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan);
nodesListAppend(dag->pSubplans, (SNode *)merge);
nodesListAppend(dag->pSubplans, (SNode *)scan);
(void)nodesListAppend(dag->pSubplans, (SNode *)merge);
(void)nodesListAppend(dag->pSubplans, (SNode *)scan);
}
void schtFreeQueryDag(SQueryPlan *dag) {}
@ -267,10 +336,22 @@ void schtBuildInsertDag(SQueryPlan *dag) {
dag->queryId = qId;
dag->numOfSubplans = 2;
dag->pSubplans = nodesMakeList();
if (NULL == dag->pSubplans) {
return;
}
SNodeListNode *inserta = (SNodeListNode *)nodesMakeNode(QUERY_NODE_NODE_LIST);
if (NULL == inserta) {
return;
}
inserta->pNodeList = nodesMakeList();
if (NULL == inserta->pNodeList) {
return;
}
SSubplan *insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
if (NULL == insertPlan) {
return;
}
insertPlan->id.queryId = qId;
insertPlan->id.groupId = 0x0000000000000003;
@ -286,13 +367,22 @@ void schtBuildInsertDag(SQueryPlan *dag) {
insertPlan->pParents = NULL;
insertPlan->pNode = NULL;
insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
if (NULL == insertPlan->pDataSink) {
return;
}
((SDataInserterNode *)insertPlan->pDataSink)->size = 1;
((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) {
return;
}
insertPlan->msgType = TDMT_VND_SUBMIT;
nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
(void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
insertPlan = (SSubplan *)nodesMakeNode(QUERY_NODE_PHYSICAL_SUBPLAN);
if (NULL == insertPlan) {
return;
}
insertPlan->id.queryId = qId;
insertPlan->id.groupId = 0x0000000000000003;
@ -308,22 +398,31 @@ void schtBuildInsertDag(SQueryPlan *dag) {
insertPlan->pParents = NULL;
insertPlan->pNode = NULL;
insertPlan->pDataSink = (SDataSinkNode *)nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_INSERT);
if (NULL == insertPlan->pDataSink) {
return;
}
((SDataInserterNode *)insertPlan->pDataSink)->size = 1;
((SDataInserterNode *)insertPlan->pDataSink)->pData = taosMemoryCalloc(1, 1);
if (NULL == ((SDataInserterNode *)insertPlan->pDataSink)->pData) {
return;
}
insertPlan->msgType = TDMT_VND_SUBMIT;
nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
(void)nodesListAppend(inserta->pNodeList, (SNode *)insertPlan);
nodesListAppend(dag->pSubplans, (SNode *)inserta);
(void)nodesListAppend(dag->pSubplans, (SNode *)inserta);
}
int32_t schtPlanToString(const SSubplan *subplan, char **str, int32_t *len) {
*str = (char *)taosMemoryCalloc(1, 20);
if (NULL == *str) {
return -1;
}
*len = 20;
return 0;
}
void schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) {}
int32_t schtExecNode(SSubplan *subplan, uint64_t groupId, SQueryNodeAddr *ep) { return 0; }
void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {}
@ -431,24 +530,27 @@ void *schtSendRsp(void *param) {
taosMsleep(1);
}
pJob = schAcquireJob(job);
code = schAcquireJob(job, &pJob);
if (code) {
return NULL;
}
void *pIter = taosHashIterate(pJob->execTasks, NULL);
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
SDataBuf msg = {0};
void *rmsg = NULL;
schtBuildSubmitRspMsg(&msg.len, &rmsg);
(void)schtBuildSubmitRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_VND_SUBMIT_RSP;
msg.pData = rmsg;
schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
(void)schHandleResponseMsg(pJob, task, task->execId, &msg, 0);
pIter = taosHashIterate(pJob->execTasks, pIter);
}
schReleaseJob(job);
(void)schReleaseJob(job);
schtJobDone = true;
@ -457,20 +559,25 @@ void *schtSendRsp(void *param) {
void *schtCreateFetchRspThread(void *param) {
int64_t job = *(int64_t *)param;
SSchJob *pJob = schAcquireJob(job);
SSchJob *pJob = NULL;
(void)schAcquireJob(job, &pJob);
if (NULL == pJob) {
return NULL;
}
taosSsleep(1);
int32_t code = 0;
SDataBuf msg = {0};
void *rmsg = NULL;
schtBuildFetchRspMsg(&msg.len, &rmsg);
(void)schtBuildFetchRspMsg(&msg.len, &rmsg);
msg.msgType = TDMT_SCH_MERGE_FETCH_RSP;
msg.pData = rmsg;
code = schHandleResponseMsg(pJob, pJob->fetchTask, pJob->fetchTask->execId, &msg, 0);
schReleaseJob(job);
(void)schReleaseJob(job);
assert(code == 0);
return NULL;
@ -488,12 +595,17 @@ void *schtFetchRspThread(void *aa) {
taosUsleep(100);
param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param));
if (NULL == param) {
return NULL;
}
param->queryId = schtQueryId;
param->taskId = schtFetchTaskId;
int32_t code = 0;
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, sizeof(SRetrieveTableRsp));
if (NULL == rsp) {
return NULL;
}
rsp->completed = 1;
rsp->numOfRows = 10;
@ -549,12 +661,17 @@ void *schtRunJobThread(void *aa) {
schtBuildQueryDag(dag);
SArray *qnodeList = taosArrayInit(1, sizeof(SQueryNodeLoad));
if (NULL == qnodeList) {
assert(0);
}
SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load);
if (NULL == taosArrayPush(qnodeList, &load)) {
assert(0);
}
queryDone = 0;
@ -572,7 +689,9 @@ void *schtRunJobThread(void *aa) {
code = schedulerExecJob(&req, &queryJobRefId);
assert(code == 0);
pJob = schAcquireJob(queryJobRefId);
pJob = NULL;
code = schAcquireJob(queryJobRefId, &pJob);
if (NULL == pJob) {
taosArrayDestroy(qnodeList);
schtFreeQueryDag(dag);
@ -580,16 +699,24 @@ void *schtRunJobThread(void *aa) {
}
execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
if (NULL == execTasks) {
assert(0);
}
void *pIter = taosHashIterate(pJob->execTasks, NULL);
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
schtFetchTaskId = task->taskId - 1;
taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task));
if (taosHashPut(execTasks, &task->taskId, sizeof(task->taskId), task, sizeof(*task))) {
assert(0);
}
pIter = taosHashIterate(pJob->execTasks, pIter);
}
param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param));
if (NULL == param) {
assert(0);
}
param->refId = queryJobRefId;
param->queryId = pJob->queryId;
@ -601,7 +728,9 @@ void *schtRunJobThread(void *aa) {
SDataBuf msg = {0};
void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
if (schtBuildQueryRspMsg(&msg.len, &rmsg)) {
assert(0);
}
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
@ -612,6 +741,9 @@ void *schtRunJobThread(void *aa) {
}
param = (SSchTaskCallbackParam *)taosMemoryCalloc(1, sizeof(*param));
if (NULL == param) {
assert(0);
}
param->refId = queryJobRefId;
param->queryId = pJob->queryId;
@ -622,7 +754,9 @@ void *schtRunJobThread(void *aa) {
param->taskId = task->taskId - 1;
SDataBuf msg = {0};
void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg);
if (schtBuildQueryRspMsg(&msg.len, &rmsg)) {
assert(0);
}
msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg;
@ -728,7 +862,9 @@ TEST(queryTest, normalCase) {
code = schedulerExecJob(&req, &job);
ASSERT_EQ(code, 0);
SSchJob *pJob = schAcquireJob(job);
SSchJob *pJob = NULL;
code = schAcquireJob(job, &pJob);
ASSERT_EQ(code, 0);
void *pIter = taosHashIterate(pJob->execTasks, NULL);
while (pIter) {
@ -839,8 +975,10 @@ TEST(queryTest, readyFirstCase) {
code = schedulerExecJob(&req, &job);
ASSERT_EQ(code, 0);
SSchJob *pJob = schAcquireJob(job);
SSchJob *pJob = NULL;
code = schAcquireJob(job, &pJob);
ASSERT_EQ(code, 0);
void *pIter = taosHashIterate(pJob->execTasks, NULL);
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
@ -956,7 +1094,9 @@ TEST(queryTest, flowCtrlCase) {
code = schedulerExecJob(&req, &job);
ASSERT_EQ(code, 0);
SSchJob *pJob = schAcquireJob(job);
SSchJob *pJob = NULL;
code = schAcquireJob(job, &pJob);
ASSERT_EQ(code, 0);
while (!queryDone) {
void *pIter = taosHashIterate(pJob->execTasks, NULL);
@ -1094,13 +1234,17 @@ TEST(otherTest, otherCase) {
schReleaseJob(0);
schFreeRpcCtx(NULL);
ASSERT_EQ(schDumpEpSet(NULL), (char *)NULL);
char* ep = NULL;
ASSERT_EQ(schDumpEpSet(NULL, &ep), TSDB_CODE_SUCCESS);
ASSERT_EQ(strcmp(schGetOpStr(SCH_OP_NULL), "NULL"), 0);
ASSERT_EQ(strcmp(schGetOpStr((SCH_OP_TYPE)100), "UNKNOWN"), 0);
}
int main(int argc, char **argv) {
schtInitLogFile();
if (rpcInit()) {
assert(0);
}
taosSeedRand(taosGetTimestampSec());
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();

2
source/libs/stream/src/streamCheckStatus.c
View File

@ -310,7 +310,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
taosArrayDestroy(pInfo->pList);
pInfo->pList = NULL;
if (pInfo->checkRspTmr != NULL) {
/*bool ret = */ taosTmrStop(pInfo->checkRspTmr);
pInfo->checkRspTmr = NULL;

302
source/libs/stream/src/streamUpdate.c
View File

@ -18,6 +18,7 @@
#include "tencode.h"
#include "tstreamUpdate.h"
#include "ttime.h"
#include "tutil.h"
#define DEFAULT_FALSE_POSITIVE 0.01
#define DEFAULT_BUCKET_SIZE 131072
@ -34,7 +35,8 @@
static int64_t adjustExpEntries(int64_t entries) { return TMIN(DEFAULT_EXPECTED_ENTRIES, entries); }
int compareKeyTs(void* pTs1, void* pTs2, void* pPkVal, __compar_fn_t cmpPkFn) {
return compareInt64Val(pTs1, pTs2);;
return compareInt64Val(pTs1, pTs2);
;
}
int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpPkFn) {
@ -49,7 +51,7 @@ int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpP
int32_t getKeyBuff(TSKEY ts, int64_t tbUid, void* pVal, int32_t len, char* buff) {
*(TSKEY*)buff = ts;
memcpy(buff+ sizeof(TSKEY), &tbUid, sizeof(int64_t));
memcpy(buff + sizeof(TSKEY), &tbUid, sizeof(int64_t));
if (len == 0) {
return sizeof(TSKEY) + sizeof(int64_t);
}
@ -66,26 +68,40 @@ int32_t getValueBuff(TSKEY ts, char* pVal, int32_t len, char* buff) {
return sizeof(TSKEY) + len;
}
void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) {
int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pInfo->numSBFs < count) {
count = pInfo->numSBFs;
}
for (uint64_t i = 0; i < count; ++i) {
int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND);
SScalableBf *tsSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE);
taosArrayPush(pInfo->pTsSBFs, &tsSBF);
SScalableBf* tsSBF = NULL;
code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &tsSBF);
TSDB_CHECK_CODE(code, lino, _error);
void* res = taosArrayPush(pInfo->pTsSBFs, &tsSBF);
if (!res) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
}
_error:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static void clearItemHelper(void *p) {
SScalableBf **pBf = p;
static void clearItemHelper(void* p) {
SScalableBf** pBf = p;
tScalableBfDestroy(*pBf);
}
void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) {
void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count) {
if (count < pInfo->numSBFs) {
for (uint64_t i = 0; i < count; ++i) {
SScalableBf *pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0);
SScalableBf* pTsSBFs = taosArrayGetP(pInfo->pTsSBFs, 0);
tScalableBfDestroy(pTsSBFs);
taosArrayRemove(pInfo->pTsSBFs, 0);
}
@ -124,14 +140,19 @@ static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t w
return watermark;
}
SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) {
return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen);
int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo) {
return updateInfoInit(pInterval->interval, pInterval->precision, watermark, igUp, pkType, pkLen, ppInfo);
}
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen) {
SUpdateInfo *pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SUpdateInfo* pInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
if (pInfo == NULL) {
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pInfo->pTsBuckets = NULL;
pInfo->pTsSBFs = NULL;
@ -145,17 +166,19 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma
bfSize = (uint64_t)(pInfo->watermark / pInfo->interval);
pInfo->numSBFs = bfSize;
pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void *));
pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void*));
if (pInfo->pTsSBFs == NULL) {
updateInfoDestroy(pInfo);
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
windowSBfAdd(pInfo, bfSize);
pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY));
if (pInfo->pTsBuckets == NULL) {
updateInfoDestroy(pInfo);
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
TSKEY dumy = 0;
@ -167,31 +190,54 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma
}
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT);
pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK);
if (!pInfo->pMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pInfo->maxDataVersion = 0;
pInfo->pkColLen = pkLen;
pInfo->pkColType = pkType;
pInfo->pKeyBuff = taosMemoryCalloc(1, sizeof(TSKEY) + sizeof(int64_t) + pkLen);
if (!pInfo->pKeyBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pkLen);
if (!pInfo->pValueBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
if (pkLen != 0) {
pInfo->comparePkRowFn = compareKeyTsAndPk;
pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC);;
pInfo->comparePkCol = getKeyComparFunc(pkType, TSDB_ORDER_ASC);
;
} else {
pInfo->comparePkRowFn = compareKeyTs;
pInfo->comparePkCol = NULL;
}
return pInfo;
(*ppInfo) = pInfo;
_end:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) {
static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (ts <= 0) {
return NULL;
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
if (pInfo->minTS < 0) {
pInfo->minTS = (TSKEY)(ts / pInfo->interval * pInfo->interval);
}
int64_t index = (int64_t)((ts - pInfo->minTS) / pInfo->interval);
if (index < 0) {
return NULL;
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
if (index >= pInfo->numSBFs) {
uint64_t count = index + 1 - pInfo->numSBFs;
@ -199,22 +245,31 @@ static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) {
windowSBfAdd(pInfo, count);
index = pInfo->numSBFs - 1;
}
SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index);
SScalableBf* res = taosArrayGetP(pInfo->pTsSBFs, index);
if (res == NULL) {
int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND);
res = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE);
code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res);
TSDB_CHECK_CODE(code, lino, _end);
taosArrayPush(pInfo->pTsSBFs, &res);
}
return res;
(*ppSBf) = res;
_end:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid) {
void *pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t));
bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) {
void* pVal = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t));
if (pVal || taosHashGetSize(pInfo->pMap) >= DEFAULT_MAP_SIZE) return true;
return false;
}
TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) {
TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) {
int32_t code = TSDB_CODE_SUCCESS;
if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN;
TSKEY maxTs = INT64_MIN;
void* pPkVal = NULL;
@ -223,14 +278,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p
int32_t len = 0;
int64_t tbUid = pBlock->info.id.uid;
SColumnInfoData *pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol);
SColumnInfoData *pPkDataInfo = NULL;
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol);
SColumnInfoData* pPkDataInfo = NULL;
if (primaryKeyCol >= 0) {
pPkDataInfo = taosArrayGet(pBlock->pDataBlock, primaryKeyCol);
}
for (int32_t i = 0; i < pBlock->info.rows; i++) {
TSKEY ts = ((TSKEY *)pColDataInfo->pData)[i];
TSKEY ts = ((TSKEY*)pColDataInfo->pData)[i];
if (maxTs < ts) {
maxTs = ts;
if (primaryKeyCol >= 0) {
@ -238,17 +293,22 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p
maxLen = colDataGetRowLength(pPkDataInfo, i);
}
}
SScalableBf *pSBf = getSBf(pInfo, ts);
SScalableBf* pSBf = NULL;
int32_t code = getSBf(pInfo, ts, &pSBf);
if (code != TSDB_CODE_SUCCESS) {
uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
}
if (pSBf) {
if (primaryKeyCol >= 0) {
pPkVal = colDataGetData(pPkDataInfo, i);
len = colDataGetRowLength(pPkDataInfo, i);
}
int32_t buffLen = getKeyBuff(ts, tbUid, pPkVal, len, pInfo->pKeyBuff);
// we don't care whether the data is updated or not
tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen);
}
}
void *pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t));
void* pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t));
if (pMaxTs == NULL || pInfo->comparePkRowFn(pMaxTs, &maxTs, pMaxPkVal, pInfo->comparePkCol) == -1) {
int32_t valueLen = getValueBuff(maxTs, pMaxPkVal, maxLen, pInfo->pValueBuff);
taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen);
@ -256,14 +316,14 @@ TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t p
return maxTs;
}
bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
int32_t res = TSDB_CODE_FAILED;
int32_t buffLen = 0;
buffLen = getKeyBuff(ts, tableId, pPkVal, len, pInfo->pKeyBuff);
void* *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t));
void** pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t));
uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets;
TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index);
TSKEY maxTs = *(TSKEY*)taosArrayGet(pInfo->pTsBuckets, index);
if (ts < maxTs - pInfo->watermark) {
// this window has been closed.
if (pInfo->pCloseWinSBF) {
@ -277,10 +337,15 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p
return true;
}
SScalableBf *pSBf = getSBf(pInfo, ts);
SScalableBf* pSBf = NULL;
int32_t code = getSBf(pInfo, ts, &pSBf);
if (code != TSDB_CODE_SUCCESS) {
uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
}
int32_t size = taosHashGetSize(pInfo->pMap);
if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1 )) {
if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) ||
(pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1)) {
int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff);
taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen);
// pSBf may be a null pointer
@ -309,7 +374,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* p
return true;
}
void updateInfoDestroy(SUpdateInfo *pInfo) {
void updateInfoDestroy(SUpdateInfo* pInfo) {
if (pInfo == NULL) {
return;
}
@ -317,7 +382,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) {
uint64_t size = taosArrayGetSize(pInfo->pTsSBFs);
for (uint64_t i = 0; i < size; i++) {
SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i);
SScalableBf* pSBF = taosArrayGetP(pInfo->pTsSBFs, i);
tScalableBfDestroy(pSBF);
}
@ -329,15 +394,19 @@ void updateInfoDestroy(SUpdateInfo *pInfo) {
taosMemoryFree(pInfo);
}
void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) {
void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo) {
if (pInfo->pCloseWinSBF) {
return;
}
int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND);
pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE);
int32_t code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &pInfo->pCloseWinSBF);
if (code != TSDB_CODE_SUCCESS) {
pInfo->pCloseWinSBF = NULL;
uError("%s failed to add close window SBF since %s", __func__, tstrerror(code));
}
}
void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) {
void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo) {
if (!pInfo || !pInfo->pCloseWinSBF) {
return;
}
@ -345,62 +414,124 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) {
pInfo->pCloseWinSBF = NULL;
}
int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) {
int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (!pInfo) {
return 0;
return TSDB_CODE_SUCCESS;
}
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tStartEncode(&encoder) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
int32_t size = taosArrayGetSize(pInfo->pTsBuckets);
if (tEncodeI32(&encoder, size) < 0) return -1;
for (int32_t i = 0; i < size; i++) {
TSKEY *pTs = (TSKEY *)taosArrayGet(pInfo->pTsBuckets, i);
if (tEncodeI64(&encoder, *pTs) < 0) return -1;
if (tEncodeI32(&encoder, size) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) return -1;
for (int32_t i = 0; i < size; i++) {
TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->pTsBuckets, i);
if (tEncodeI64(&encoder, *pTs) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
}
if (tEncodeU64(&encoder, pInfo->numBuckets) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
int32_t sBfSize = taosArrayGetSize(pInfo->pTsSBFs);
if (tEncodeI32(&encoder, sBfSize) < 0) return -1;
if (tEncodeI32(&encoder, sBfSize) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
for (int32_t i = 0; i < sBfSize; i++) {
SScalableBf *pSBf = taosArrayGetP(pInfo->pTsSBFs, i);
if (tScalableBfEncode(pSBf, &encoder) < 0) return -1;
SScalableBf* pSBf = taosArrayGetP(pInfo->pTsSBFs, i);
if (tScalableBfEncode(pSBf, &encoder) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
}
if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) return -1;
if (tEncodeI64(&encoder, pInfo->interval) < 0) return -1;
if (tEncodeI64(&encoder, pInfo->watermark) < 0) return -1;
if (tEncodeI64(&encoder, pInfo->minTS) < 0) return -1;
if (tEncodeU64(&encoder, pInfo->numSBFs) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
if (tEncodeI64(&encoder, pInfo->interval) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
if (tEncodeI64(&encoder, pInfo->watermark) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
if (tEncodeI64(&encoder, pInfo->minTS) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) return -1;
if (tScalableBfEncode(pInfo->pCloseWinSBF, &encoder) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
int32_t mapSize = taosHashGetSize(pInfo->pMap);
if (tEncodeI32(&encoder, mapSize) < 0) return -1;
void *pIte = NULL;
if (tEncodeI32(&encoder, mapSize) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
void* pIte = NULL;
size_t keyLen = 0;
while ((pIte = taosHashIterate(pInfo->pMap, pIte)) != NULL) {
void *key = taosHashGetKey(pIte, &keyLen);
if (tEncodeU64(&encoder, *(uint64_t *)key) < 0) return -1;
void* key = taosHashGetKey(pIte, &keyLen);
if (tEncodeU64(&encoder, *(uint64_t*)key) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
int32_t valueSize = taosHashGetValueSize(pIte);
if (tEncodeBinary(&encoder, (const uint8_t *)pIte, valueSize) < 0) return -1;
if (tEncodeBinary(&encoder, (const uint8_t*)pIte, valueSize) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
}
if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) return -1;
if (tEncodeU64(&encoder, pInfo->maxDataVersion) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) return -1;
if (tEncodeI8(&encoder, pInfo->pkColType) < 0) return -1;
if (tEncodeI32(&encoder, pInfo->pkColLen) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
if (tEncodeI8(&encoder, pInfo->pkColType) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _end);
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
*pLen = tlen;
_end:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) {
int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
ASSERT(pInfo);
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
@ -419,10 +550,12 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) {
int32_t sBfSize = 0;
if (tDecodeI32(&decoder, &sBfSize) < 0) return -1;
pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void *));
pInfo->pTsSBFs = taosArrayInit(sBfSize, sizeof(void*));
for (int32_t i = 0; i < sBfSize; i++) {
SScalableBf *pSBf = tScalableBfDecode(&decoder);
if (!pSBf) return -1;
SScalableBf* pSBf = NULL;
code = tScalableBfDecode(&decoder, &pSBf);
TSDB_CHECK_CODE(code, lino, _error);
taosArrayPush(pInfo->pTsSBFs, &pSBf);
}
@ -430,15 +563,20 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) {
if (tDecodeI64(&decoder, &pInfo->interval) < 0) return -1;
if (tDecodeI64(&decoder, &pInfo->watermark) < 0) return -1;
if (tDecodeI64(&decoder, &pInfo->minTS) < 0) return -1;
pInfo->pCloseWinSBF = tScalableBfDecode(&decoder);
code = tScalableBfDecode(&decoder, &pInfo->pCloseWinSBF);
if (code != TSDB_CODE_SUCCESS) {
pInfo->pCloseWinSBF = NULL;
code = TSDB_CODE_SUCCESS;
}
int32_t mapSize = 0;
if (tDecodeI32(&decoder, &mapSize) < 0) return -1;
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT);
pInfo->pMap = taosHashInit(mapSize, hashFn, true, HASH_NO_LOCK);
uint64_t uid = 0;
void* pVal = NULL;
int32_t valSize = 0;
void* pVal = NULL;
int32_t valSize = 0;
for (int32_t i = 0; i < mapSize; i++) {
if (tDecodeU64(&decoder, &uid) < 0) return -1;
if (tDecodeBinary(&decoder, (uint8_t**)&pVal, &valSize) < 0) return -1;
@ -454,7 +592,8 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) {
pInfo->pValueBuff = taosMemoryCalloc(1, sizeof(TSKEY) + pInfo->pkColLen);
if (pInfo->pkColLen != 0) {
pInfo->comparePkRowFn = compareKeyTsAndPk;
pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC);;
pInfo->comparePkCol = getKeyComparFunc(pInfo->pkColType, TSDB_ORDER_ASC);
;
} else {
pInfo->comparePkRowFn = compareKeyTs;
pInfo->comparePkCol = NULL;
@ -463,11 +602,16 @@ int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo) {
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
_error:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
TSKEY *pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t));
bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) {
TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t));
bool res = true;
if (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == 1) {
res = false;

14
source/libs/sync/inc/syncInt.h
View File

@ -74,16 +74,16 @@ typedef struct SRaftStore {
TdThreadMutex mutex;
} SRaftStore;
typedef struct SSyncHbTimerData {
struct SSyncHbTimerData {
int64_t syncNodeRid;
SSyncTimer* pTimer;
SRaftId destId;
uint64_t logicClock;
int64_t execTime;
int64_t rid;
} SSyncHbTimerData;
};
typedef struct SSyncTimer {
struct SSyncTimer {
void* pTimer;
TAOS_TMR_CALLBACK timerCb;
uint64_t logicClock;
@ -92,7 +92,7 @@ typedef struct SSyncTimer {
int64_t timeStamp;
SRaftId destId;
int64_t hbDataRid;
} SSyncTimer;
};
typedef struct SElectTimerParam {
uint64_t logicClock;
@ -106,7 +106,7 @@ typedef struct SPeerState {
int64_t lastSendTime;
} SPeerState;
typedef struct SSyncNode {
struct SSyncNode {
// init by SSyncInfo
SyncGroupId vgId;
SRaftCfg raftCfg;
@ -116,7 +116,7 @@ typedef struct SSyncNode {
// sync io
SSyncLogBuffer* pLogBuf;
SWal* pWal;
struct SWal* pWal;
const SMsgCb* msgcb;
int32_t (*syncSendMSg)(const SEpSet* pEpSet, SRpcMsg* pMsg);
int32_t (*syncEqMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg);
@ -234,7 +234,7 @@ typedef struct SSyncNode {
bool isStart;
} SSyncNode;
};
// open/close --------------
SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo, int32_t vnodeVersion);

2
source/libs/transport/inc/transComm.h
View File

@ -103,7 +103,7 @@ typedef void* queue[2];
#define TRANS_MAGIC_NUM 0x5f375a86
#define TRANS_NOVALID_PACKET(src) ((src) != TRANS_MAGIC_NUM ? 1 : 0)
typedef SRpcMsg STransMsg;
typedef struct SRpcMsg STransMsg;
typedef SRpcCtx STransCtx;
typedef SRpcCtxVal STransCtxVal;
typedef SRpcInfo STrans;

37
source/util/src/talgo.c
View File

@ -147,10 +147,14 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, cons
}
}
void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) {
int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) {
char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer
if (NULL == buf) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf);
taosMemoryFreeClear(buf);
return 0;
}
#define DOSWAP(a, b, size) \
@ -324,14 +328,17 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size
}
}
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char *buf, bool maxroot) {
int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char *buf, bool maxroot) {
int32_t parent;
int32_t child;
char *tmp = NULL;
if (buf == NULL) {
tmp = taosMemoryMalloc(size);
if (NULL == tmp) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
tmp = buf;
}
@ -378,24 +385,31 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
if (buf == NULL) {
taosMemoryFree(tmp);
}
return 0;
}
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
bool maxroot) {
int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
bool maxroot) {
int32_t i;
char *buf = taosMemoryCalloc(1, size);
if (buf == NULL) {
return;
return TSDB_CODE_OUT_OF_MEMORY;
}
if (base && size > 0) {
for (i = len / 2 - 1; i >= 0; i--) {
taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot);
int32_t code = taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot);
if (code) {
taosMemoryFree(buf);
return code;
}
}
}
taosMemoryFree(buf);
return 0;
}
static void taosMerge(void *src, int32_t start, int32_t leftend, int32_t end, int64_t size, const void *param,
@ -441,7 +455,10 @@ static int32_t taosMergeSortHelper(void *src, int64_t numOfElem, int64_t size, c
// short array sort, instead of merge sort process
const int32_t THRESHOLD_SIZE = 6;
char *buf = taosMemoryCalloc(1, size); // prepare the swap buffer
if (buf == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (buf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t start = 0; start < numOfElem - 1; start += THRESHOLD_SIZE) {
int32_t end = (start + THRESHOLD_SIZE - 1) <= numOfElem - 1 ? (start + THRESHOLD_SIZE - 1) : numOfElem - 1;
tInsertSort(src, size, start, end, param, comparFn, buf);
@ -451,7 +468,9 @@ static int32_t taosMergeSortHelper(void *src, int64_t numOfElem, int64_t size, c
if (numOfElem > THRESHOLD_SIZE) {
int32_t currSize;
void *tmp = taosMemoryMalloc(numOfElem * size);
if (tmp == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (currSize = THRESHOLD_SIZE; currSize <= numOfElem - 1; currSize = 2 * currSize) {
int32_t leftStart;

5
source/util/src/tarray.c
View File

@ -518,9 +518,8 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t
// todo remove it
// order array<type *>
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) {
taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn);
// taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param);
int32_t taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) {
return taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn);
}
void taosArraySwap(SArray* a, SArray* b) {

94
source/util/src/tbloomfilter.c
View File

@ -22,26 +22,30 @@
#define UNIT_NUM_BITS 64ULL
#define UNIT_ADDR_NUM_BITS 6ULL
static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) {
static FORCE_INLINE bool setBit(uint64_t* buf, uint64_t index) {
uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
uint64_t old = buf[unitIndex];
buf[unitIndex] |= (1ULL << (index % UNIT_NUM_BITS));
return buf[unitIndex] != old;
}
static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) {
static FORCE_INLINE bool getBit(uint64_t* buf, uint64_t index) {
uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS;
uint64_t mask = 1ULL << (index % UNIT_NUM_BITS);
return buf[unitIndex] & mask;
}
SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) {
int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF) {
int32_t code = 0;
int32_t lino = 0;
if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) {
return NULL;
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
if (pBF == NULL) {
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pBF->expectedEntries = expectedEntries;
pBF->errorRate = errorRate;
@ -61,12 +65,19 @@ SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate) {
pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t));
if (pBF->buffer == NULL) {
tBloomFilterDestroy(pBF);
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
return pBF;
(*ppBF) = pBF;
_error:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) {
int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) {
ASSERT(!tBloomFilterIsFull(pBF));
bool hasChange = false;
const register uint64_t size = pBF->numBits;
@ -82,7 +93,7 @@ int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) {
return TSDB_CODE_FAILED;
}
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) {
int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len) {
uint64_t h1 = (uint64_t)pBF->hashFn1(keyBuf, len);
uint64_t h2 = (uint64_t)pBF->hashFn2(keyBuf, len);
bool hasChange = false;
@ -99,7 +110,7 @@ int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len) {
return TSDB_CODE_FAILED;
}
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t hash2) {
int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t hash1, uint64_t hash2) {
const register uint64_t size = pBF->numBits;
uint64_t cbHash = hash1;
for (uint32_t i = 0; i < pBF->hashFunctions; ++i) {
@ -111,7 +122,7 @@ int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t hash1, uint64_t
return TSDB_CODE_FAILED;
}
void tBloomFilterDestroy(SBloomFilter *pBF) {
void tBloomFilterDestroy(SBloomFilter* pBF) {
if (pBF == NULL) {
return;
}
@ -119,41 +130,70 @@ void tBloomFilterDestroy(SBloomFilter *pBF) {
taosMemoryFree(pBF);
}
int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder) {
int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder) {
if (tEncodeU32(pEncoder, pBF->hashFunctions) < 0) return -1;
if (tEncodeU64(pEncoder, pBF->expectedEntries) < 0) return -1;
if (tEncodeU64(pEncoder, pBF->numUnits) < 0) return -1;
if (tEncodeU64(pEncoder, pBF->numBits) < 0) return -1;
if (tEncodeU64(pEncoder, pBF->size) < 0) return -1;
for (uint64_t i = 0; i < pBF->numUnits; i++) {
uint64_t *pUnits = (uint64_t *)pBF->buffer;
uint64_t* pUnits = (uint64_t*)pBF->buffer;
if (tEncodeU64(pEncoder, pUnits[i]) < 0) return -1;
}
if (tEncodeDouble(pEncoder, pBF->errorRate) < 0) return -1;
return 0;
}
SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder) {
SBloomFilter *pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF) {
int32_t code = 0;
int32_t lino = 0;
SBloomFilter* pBF = taosMemoryCalloc(1, sizeof(SBloomFilter));
if (!pBF) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pBF->buffer = NULL;
if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) goto _error;
if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) goto _error;
if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) goto _error;
if (tDecodeU64(pDecoder, &pBF->numBits) < 0) goto _error;
if (tDecodeU64(pDecoder, &pBF->size) < 0) goto _error;
if (tDecodeU32(pDecoder, &pBF->hashFunctions) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
if (tDecodeU64(pDecoder, &pBF->expectedEntries) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
if (tDecodeU64(pDecoder, &pBF->numUnits) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
if (tDecodeU64(pDecoder, &pBF->numBits) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
if (tDecodeU64(pDecoder, &pBF->size) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
pBF->buffer = taosMemoryCalloc(pBF->numUnits, sizeof(uint64_t));
for (int32_t i = 0; i < pBF->numUnits; i++) {
uint64_t *pUnits = (uint64_t *)pBF->buffer;
if (tDecodeU64(pDecoder, pUnits + i) < 0) goto _error;
uint64_t* pUnits = (uint64_t*)pBF->buffer;
if (tDecodeU64(pDecoder, pUnits + i) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
}
if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
if (tDecodeDouble(pDecoder, &pBF->errorRate) < 0) goto _error;
pBF->hashFn1 = HASH_FUNCTION_1;
pBF->hashFn2 = HASH_FUNCTION_2;
return pBF;
(*ppBF) = pBF;
return TSDB_CODE_SUCCESS;
_error:
tBloomFilterDestroy(pBF);
return NULL;
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return code;
}
bool tBloomFilterIsFull(const SBloomFilter *pBF) { return pBF->size >= pBF->expectedEntries; }
bool tBloomFilterIsFull(const SBloomFilter* pBF) { return pBF->size >= pBF->expectedEntries; }

16
source/util/src/tencode.c
View File

@ -72,10 +72,14 @@ int32_t tStartEncode(SEncoder* pCoder) {
SEncoderNode* pNode;
if (pCoder->data) {
if (pCoder->size - pCoder->pos < sizeof(int32_t)) return -1;
if (pCoder->size - pCoder->pos < sizeof(int32_t)) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
pNode = tEncoderMalloc(pCoder, sizeof(*pNode));
if (pNode == NULL) return -1;
if (pNode == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
pNode->data = pCoder->data;
pNode->pos = pCoder->pos;
@ -111,7 +115,7 @@ void tEndEncode(SEncoder* pCoder) {
(void)tEncodeI32(pCoder, len);
TD_CODER_MOVE_POS(pCoder, len);
pCoder->pos += len;
}
}
@ -119,10 +123,12 @@ int32_t tStartDecode(SDecoder* pCoder) {
SDecoderNode* pNode;
int32_t len;
if (tDecodeI32(pCoder, &len) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI32(pCoder, &len));
pNode = tDecoderMalloc(pCoder, sizeof(*pNode));
if (pNode == NULL) return -1;
if (pNode == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
pNode->data = pCoder->data;
pNode->pos = pCoder->pos;

4
source/util/src/terror.c
View File

@ -789,6 +789,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR, "Invalid TDLite open
TAOS_DEFINE_ERROR(TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY, "Queue out of memory")
//AUDIT
TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON, "can't format to json")
TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD, "Failed to send out audit record")
TAOS_DEFINE_ERROR(TSDB_CODE_AUDIT_FAIL_GENERATE_JSON, "Failed to generate json")
#ifdef TAOS_ERROR_C
};
#endif

191
source/util/src/tscalablebf.c
View File

@ -17,6 +17,7 @@
#include "tscalablebf.h"
#include "taoserror.h"
#include "tutil.h"
#define DEFAULT_GROWTH 2
#define DEFAULT_TIGHTENING_RATIO 0.5
@ -24,82 +25,120 @@
#define SBF_INVALID -1
#define SBF_VALID 0
static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate);
static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate,
SBloomFilter** ppNormalBf);
SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) {
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
const uint32_t defaultSize = 8;
if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) {
return NULL;
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
if (pSBf == NULL) {
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS;
pSBf->status = SBF_VALID;
pSBf->numBits = 0;
pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *));
if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) {
pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void*));
if (!pSBf->bfArray) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
SBloomFilter* pNormalBf = NULL;
code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf);
if (code != TSDB_CODE_SUCCESS) {
tScalableBfDestroy(pSBf);
return NULL;
TSDB_CHECK_CODE(code, lino, _error);
}
pSBf->growth = DEFAULT_GROWTH;
pSBf->hashFn1 = HASH_FUNCTION_1;
pSBf->hashFn2 = HASH_FUNCTION_2;
return pSBf;
(*ppSBf) = pSBf;
return TSDB_CODE_SUCCESS;
_error:
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return code;
}
int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pSBf->status == SBF_INVALID) {
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
int32_t size = taosArrayGetSize(pSBf->bfArray);
SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
if (!pNormalBf) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
int32_t size = taosArrayGetSize(pSBf->bfArray);
SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
ASSERT(pNormalBf);
if (tBloomFilterIsFull(pNormalBf)) {
pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO);
if (pNormalBf == NULL) {
code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf);
if (code != TSDB_CODE_SUCCESS) {
pSBf->status = SBF_INVALID;
return TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
}
return tBloomFilterPut(pNormalBf, keyBuf, len);
_error:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (pSBf->status == SBF_INVALID) {
return TSDB_CODE_FAILED;
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len);
uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len);
int32_t size = taosArrayGetSize(pSBf->bfArray);
int32_t size = taosArrayGetSize(pSBf->bfArray);
for (int32_t i = size - 2; i >= 0; --i) {
if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
}
SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
ASSERT(pNormalBf);
if (tBloomFilterIsFull(pNormalBf)) {
pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO);
if (pNormalBf == NULL) {
code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf);
if (code != TSDB_CODE_SUCCESS) {
pSBf->status = SBF_INVALID;
return TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
}
return tBloomFilterPutHash(pNormalBf, h1, h2);
_error:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len) {
if (pSBf->status == SBF_INVALID) {
return TSDB_CODE_FAILED;
}
uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len);
uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len);
int32_t size = taosArrayGetSize(pSBf->bfArray);
int32_t size = taosArrayGetSize(pSBf->bfArray);
for (int32_t i = size - 1; i >= 0; --i) {
if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
@ -108,24 +147,35 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32
return TSDB_CODE_SUCCESS;
}
static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) {
static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate,
SBloomFilter** ppNormalBf) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) {
return NULL;
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate);
if (pNormalBf == NULL) {
return NULL;
}
SBloomFilter* pNormalBf = NULL;
code = tBloomFilterInit(expectedEntries, errorRate, &pNormalBf);
TSDB_CHECK_CODE(code, lino, _error);
if (taosArrayPush(pSBf->bfArray, &pNormalBf) == NULL) {
tBloomFilterDestroy(pNormalBf);
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pSBf->numBits += pNormalBf->numBits;
return pNormalBf;
(*ppNormalBf) = pNormalBf;
_error:
if (code != TSDB_CODE_SUCCESS) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}
void tScalableBfDestroy(SScalableBf *pSBf) {
void tScalableBfDestroy(SScalableBf* pSBf) {
if (pSBf == NULL) {
return;
}
@ -135,7 +185,7 @@ void tScalableBfDestroy(SScalableBf *pSBf) {
taosMemoryFree(pSBf);
}
int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) {
int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder) {
if (!pSBf) {
if (tEncodeI32(pEncoder, 0) < 0) return -1;
return 0;
@ -143,7 +193,7 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) {
int32_t size = taosArrayGetSize(pSBf->bfArray);
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; i++) {
SBloomFilter *pBF = taosArrayGetP(pSBf->bfArray, i);
SBloomFilter* pBF = taosArrayGetP(pSBf->bfArray, i);
if (tBloomFilterEncode(pBF, pEncoder) < 0) return -1;
}
if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1;
@ -153,30 +203,61 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) {
return 0;
}
SScalableBf *tScalableBfDecode(SDecoder *pDecoder) {
SScalableBf *pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
if (!pSBf) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
pSBf->hashFn1 = HASH_FUNCTION_1;
pSBf->hashFn2 = HASH_FUNCTION_2;
pSBf->bfArray = NULL;
int32_t size = 0;
if (tDecodeI32(pDecoder, &size) < 0) goto _error;
if (size == 0) {
tScalableBfDestroy(pSBf);
return NULL;
if (tDecodeI32(pDecoder, &size) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
pSBf->bfArray = taosArrayInit(size * 2, sizeof(void *));
if (size == 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
pSBf->bfArray = taosArrayInit(size * 2, POINTER_BYTES);
if (!pSBf->bfArray) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
}
for (int32_t i = 0; i < size; i++) {
SBloomFilter *pBF = tBloomFilterDecode(pDecoder);
if (!pBF) goto _error;
SBloomFilter* pBF = NULL;
code = tBloomFilterDecode(pDecoder, &pBF);
TSDB_CHECK_CODE(code, lino, _error);
taosArrayPush(pSBf->bfArray, &pBF);
}
if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error;
if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error;
if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error;
if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error;
return pSBf;
if (tDecodeU32(pDecoder, &pSBf->growth) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
if (tDecodeI8(pDecoder, &pSBf->status) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _error);
}
(*ppSBf) = pSBf;
_error:
tScalableBfDestroy(pSBf);
return NULL;
if (code != TSDB_CODE_SUCCESS) {
tScalableBfDestroy(pSBf);
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
}

53
source/util/test/bloomFilterTest.cpp
View File

@ -8,12 +8,15 @@ using namespace std;
TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
int64_t ts1 = 1650803518000;
GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 0));
GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, 1));
GTEST_ASSERT_EQ(NULL, tBloomFilterInit(100, -0.1));
GTEST_ASSERT_EQ(NULL, tBloomFilterInit(0, 0.01));
SBloomFilter* pBFTmp = NULL;
GTEST_ASSERT_NE(0, tBloomFilterInit(100, 0, &pBFTmp));
GTEST_ASSERT_NE(0, tBloomFilterInit(100, 1, &pBFTmp));
GTEST_ASSERT_NE(0, tBloomFilterInit(100, -0.1, &pBFTmp));
GTEST_ASSERT_NE(0, tBloomFilterInit(0, 0.01, &pBFTmp));
SBloomFilter *pBF1 = tBloomFilterInit(100, 0.005);
SBloomFilter *pBF1 = NULL;
int32_t code = tBloomFilterInit(100, 0.005, &pBF1);
GTEST_ASSERT_EQ(0, code);
GTEST_ASSERT_EQ(pBF1->numBits, 1152);
GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64);
int64_t count = 0;
@ -25,16 +28,19 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
}
ASSERT_TRUE(tBloomFilterIsFull(pBF1));
SBloomFilter *pBF2 = tBloomFilterInit(1000 * 10000, 0.1);
SBloomFilter* pBF2 = NULL;
GTEST_ASSERT_EQ(0, tBloomFilterInit(1000 * 10000, 0.1, &pBF2));
GTEST_ASSERT_EQ(pBF2->numBits, 47925312);
GTEST_ASSERT_EQ(pBF2->numUnits, 47925312 / 64);
SBloomFilter *pBF3 = tBloomFilterInit(10000 * 10000, 0.001);
SBloomFilter* pBF3 = NULL;
GTEST_ASSERT_EQ(0, tBloomFilterInit(10000 * 10000, 0.001, &pBF3));
GTEST_ASSERT_EQ(pBF3->numBits, 1437758784);
GTEST_ASSERT_EQ(pBF3->numUnits, 1437758784 / 64);
int64_t size = 10000;
SBloomFilter *pBF4 = tBloomFilterInit(size, 0.001);
SBloomFilter* pBF4 = NULL;
GTEST_ASSERT_EQ(0, tBloomFilterInit(size, 0.001, &pBF4));
for (int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tBloomFilterPut(pBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);
@ -42,17 +48,17 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
ASSERT_TRUE(!tBloomFilterIsFull(pBF4));
for (int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t));
uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t));
int64_t ts = i + ts1;
uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t));
uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t));
GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_FAILED);
}
for (int64_t i = 2000; i < 3000; i++) {
int64_t ts = i + ts1;
uint64_t h1 = (uint64_t) pBF4->hashFn1((const char*)&ts, sizeof(int64_t));
uint64_t h2 = (uint64_t) pBF4->hashFn2((const char*)&ts, sizeof(int64_t));
GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS);
int64_t ts = i + ts1;
uint64_t h1 = (uint64_t)pBF4->hashFn1((const char*)&ts, sizeof(int64_t));
uint64_t h2 = (uint64_t)pBF4->hashFn2((const char*)&ts, sizeof(int64_t));
GTEST_ASSERT_EQ(tBloomFilterNoContain(pBF4, h1, h2), TSDB_CODE_SUCCESS);
}
tBloomFilterDestroy(pBF1);
@ -64,12 +70,14 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) {
TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
int64_t ts1 = 1650803518000;
GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 0));
GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, 1));
GTEST_ASSERT_EQ(NULL, tScalableBfInit(100, -0.1));
GTEST_ASSERT_EQ(NULL, tScalableBfInit(0, 0.01));
SScalableBf* tsSBF = NULL;
GTEST_ASSERT_NE(0, tScalableBfInit(100, 0, &tsSBF));
GTEST_ASSERT_NE(0, tScalableBfInit(100, 1, &tsSBF));
GTEST_ASSERT_NE(0, tScalableBfInit(100, -0.1, &tsSBF));
GTEST_ASSERT_NE(0, tScalableBfInit(0, 0.01, &tsSBF));
SScalableBf *pSBF1 = tScalableBfInit(100, 0.01);
SScalableBf* pSBF1 = NULL;
GTEST_ASSERT_EQ(0, tScalableBfInit(100, 0.01, &pSBF1));
GTEST_ASSERT_EQ(pSBF1->numBits, 1152);
int64_t count = 0;
int64_t index = 0;
@ -108,7 +116,7 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
int32_t aSize = taosArrayGetSize(pSBF1->bfArray);
int64_t totalBits = 0;
for (int64_t i = 0; i < aSize; i++) {
SBloomFilter *pBF = (SBloomFilter *)taosArrayGetP(pSBF1->bfArray, i);
SBloomFilter* pBF = (SBloomFilter*)taosArrayGetP(pSBF1->bfArray, i);
ASSERT_TRUE(tBloomFilterIsFull(pBF));
totalBits += pBF->numBits;
}
@ -120,7 +128,8 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) {
}
int64_t size = 10000;
SScalableBf *pSBF4 = tScalableBfInit(size, 0.001);
SScalableBf* pSBF4 = NULL;
GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, &pSBF4));
for (int64_t i = 0; i < 1000; i++) {
int64_t ts = i + ts1;
GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS);

21
tests/army/cmdline/fullopt.py
View File

@ -28,13 +28,13 @@ from frame import *
class TDTestCase(TBase):
updatecfgDict = {
'queryMaxConcurrentTables': '2K',
'streamMax': '1M',
'queryMaxConcurrentTables': '2K',
'streamMax': '1M',
'totalMemoryKB': '1G',
'streamMax': '1P',
'streamBufferSize':'1T',
'slowLogScope':"query"
}
}
def insertData(self):
tdLog.info(f"insert data.")
@ -53,7 +53,7 @@ class TDTestCase(TBase):
def doTaos(self):
tdLog.info(f"check taos command options...")
# local command
options = [
"DebugFlag 143",
@ -71,7 +71,6 @@ class TDTestCase(TBase):
"smlTsDefaultName tsdef",
"serverPort 6030",
"timezone tz",
"tempDir /var/tmp"
]
# exec
for option in options:
@ -83,7 +82,7 @@ class TDTestCase(TBase):
# help
rets = etool.runBinFile("taos", "--help")
self.checkListNotEmpty(rets)
# b r w s
# b r w s
sql = f"select * from {self.db}.{self.stb} limit 10"
rets = etool.runBinFile("taos", f'-B -r -w 100 -s "{sql}" ')
self.checkListNotEmpty(rets)
@ -124,16 +123,16 @@ class TDTestCase(TBase):
tdLog.info(f"check taosd command options...")
idx = 1 # dnode1
cfg = sc.dnodeCfgPath(idx)
# -s
sdb = "./sdb.json"
eos.delFile(sdb)
etool.exeBinFile("taosd", f"-s -c {cfg}")
# -C
etool.exeBinFile("taosd", "-C")
# -k
# -k
etool.exeBinFile("taosd", "-k", False)
# -V
rets = etool.runBinFile("taosd", "-V")
@ -153,7 +152,7 @@ class TDTestCase(TBase):
sc.dnodeStop(idx)
etool.exeBinFile("taosd", "-a http://192.168.1.10")
#exe
#exe
etool.exeBinFile("taosd", f"-E abc -c {cfg}", False)
sc.dnodeStop(idx)
etool.exeBinFile("taosd", f"-e def -c {cfg}", False)
@ -182,7 +181,7 @@ class TDTestCase(TBase):
# do action
self.doTaosd()
tdLog.success(f"{__file__} successfully executed")

3
tests/script/tsim/insert/update2.sim
View File

@ -120,7 +120,8 @@ endi
if $data01 != 2 then
return -1
endi
if $data02 != 1 then
if $data02 != 1 then
print expect 1 , actual $data02
return -1
endi
if $data03 != 1 then

2
tools/shell/src/shellArguments.c
View File

@ -311,7 +311,7 @@ static void shellInitArgs(int argc, char *argv[]) {
if (strlen(argv[i]) == 2) {
printf("Enter password: ");
taosSetConsoleEcho(false);
if (scanf("%20s", shell.args.password) > 1) {
if (scanf("%128s", shell.args.password) > 1) {
fprintf(stderr, "password reading error\n");
}
taosSetConsoleEcho(true);

36
utils/test/c/sml_test.c
View File

@ -121,7 +121,8 @@ int smlProcess_json1_Test() {
char *sql1[1] = {0};
for (int i = 0; i < 1; i++) {
sql1[i] = taosMemoryCalloc(1, 1024);
strncpy(sql1[i], sql[i], 1023);
ASSERT(sql1[i] != NULL);
(void)strncpy(sql1[i], sql[i], 1023);
}
pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL,
@ -149,7 +150,8 @@ int smlProcess_json1_Test() {
char *sql3[1] = {0};
for (int i = 0; i < 1; i++) {
sql3[i] = taosMemoryCalloc(1, 1024);
strncpy(sql3[i], sql2[i], 1023);
ASSERT(sql3[i] != NULL);
(void)strncpy(sql3[i], sql2[i], 1023);
}
pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL,
@ -176,7 +178,8 @@ int smlProcess_json1_Test() {
char *sql5[1] = {0};
for (int i = 0; i < 1; i++) {
sql5[i] = taosMemoryCalloc(1, 1024);
strncpy(sql5[i], sql4[i], 1023);
ASSERT(sql5[i] != NULL);
(void)strncpy(sql5[i], sql4[i], 1023);
}
pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL,
@ -215,7 +218,8 @@ int smlProcess_json2_Test() {
char *sql1[1] = {0};
for (int i = 0; i < 1; i++) {
sql1[i] = taosMemoryCalloc(1, 1024);
strncpy(sql1[i], sql[i], 1023);
ASSERT(sql1[i] != NULL);
(void)strncpy(sql1[i], sql[i], 1023);
}
pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL,
@ -249,7 +253,8 @@ int smlProcess_json3_Test() {
char *sql1[1] = {0};
for (int i = 0; i < 1; i++) {
sql1[i] = taosMemoryCalloc(1, 1024);
strncpy(sql1[i], sql[i], 1023);
ASSERT(sql1[i] != NULL);
(void)strncpy(sql1[i], sql[i], 1023);
}
pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL,
@ -286,7 +291,8 @@ int smlProcess_json_tag_not_same_Test() {
char *sql1[1] = {0};
for (int i = 0; i < 1; i++) {
sql1[i] = taosMemoryCalloc(1, 1024);
strncpy(sql1[i], sql[i], 1023);
ASSERT(sql1[i] != NULL);
(void)strncpy(sql1[i], sql[i], 1023);
}
pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL,
@ -312,7 +318,8 @@ int smlProcess_json_tag_not_same_Test() {
char *sql3[1] = {0};
for (int i = 0; i < 1; i++) {
sql3[i] = taosMemoryCalloc(1, 1024);
strncpy(sql3[i], sql2[i], 1023);
ASSERT(sql3[i] != NULL);
(void)strncpy(sql3[i], sql2[i], 1023);
}
pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL,
@ -338,7 +345,8 @@ int smlProcess_json_tag_not_same_Test() {
char *sql5[1] = {0};
for (int i = 0; i < 1; i++) {
sql5[i] = taosMemoryCalloc(1, 1024);
strncpy(sql5[i], sql4[i], 1023);
ASSERT(sql5[i] != NULL);
(void)strncpy(sql5[i], sql4[i], 1023);
}
pRes = taos_schemaless_insert(taos, (char **)sql5, sizeof(sql5) / sizeof(sql5[0]), TSDB_SML_JSON_PROTOCOL,
@ -1147,7 +1155,8 @@ int sml_19221_Test() {
taos_free_result(pRes);
char *tmp = (char *)taosMemoryCalloc(1024, 1);
memcpy(tmp, sql[0], strlen(sql[0]));
ASSERT(tmp != NULL);
(void)memcpy(tmp, sql[0], strlen(sql[0]));
*(char *)(tmp + 44) = 0;
int32_t totalRows = 0;
pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL,
@ -1218,7 +1227,8 @@ int sml_ts3116_Test() {
taos_free_result(pRes);
int32_t totalRows = 0;
char *tmp = (char *)taosMemoryCalloc(1024, 1);
memcpy(tmp, sql, strlen(sql));
ASSERT(tmp != NULL);
(void)memcpy(tmp, sql, strlen(sql));
totalRows = 0;
pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL,
TSDB_SML_TIMESTAMP_MILLI_SECONDS);
@ -1235,7 +1245,8 @@ int sml_ts3116_Test() {
taos_free_result(pRes);
tmp = (char *)taosMemoryCalloc(1024, 1);
memcpy(tmp, sql1, strlen(sql1));
ASSERT(tmp != NULL);
(void)memcpy(tmp, sql1, strlen(sql1));
totalRows = 0;
pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL,
TSDB_SML_TIMESTAMP_MILLI_SECONDS);
@ -2049,7 +2060,8 @@ int sml_td29373_Test() {
char *sql3[1] = {0};
for (int i = 0; i < 1; i++) {
sql3[i] = taosMemoryCalloc(1, 1024);
strncpy(sql3[i], sql2[i], 1023);
ASSERT(sql3[i] != NULL);
(void)strncpy(sql3[i], sql2[i], 1023);
}
pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL,