xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: refactor the module dependency.

This commit is contained in:
Haojun Liao 2023-05-23 18:29:23 +08:00
parent f520de0dab
commit e67b532d8a
44 changed files with 1477 additions and 867 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/common/tcommon.h
View File

@ -37,6 +37,13 @@ extern "C" {
) )
// clang-format on // clang-format on
typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
typedef struct STableKeyInfo {
uint64_t uid;
uint64_t groupId;
} STableKeyInfo;
typedef struct SWinKey { typedef struct SWinKey {
uint64_t groupId; uint64_t groupId;
TSKEY ts; TSKEY ts;

7
include/libs/executor/executor.h
View File

@ -23,6 +23,7 @@ extern "C" {
#include "query.h" #include "query.h"
#include "tcommon.h" #include "tcommon.h"
#include "tmsgcb.h" #include "tmsgcb.h"
#include "storageapi.h"
typedef void* qTaskInfo_t; typedef void* qTaskInfo_t;
typedef void* DataSinkHandle; typedef void* DataSinkHandle;
@ -51,10 +52,10 @@ typedef struct {
bool initTableReader; bool initTableReader;
bool initTqReader; bool initTqReader;
int32_t numOfVgroups; int32_t numOfVgroups;
void* sContext; // SSnapContext*
void* sContext; // SSnapContext* void* pStateBackend;
struct SStorageAPI api;
void* pStateBackend;
} SReadHandle; } SReadHandle;
// in queue mode, data streams are seperated by msg // in queue mode, data streams are seperated by msg

576
include/libs/executor/storageapi.h Normal file
View File

@ -0,0 +1,576 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_STORAGEAPI_H
#define TDENGINE_STORAGEAPI_H
#include "tsimplehash.h"
#include "tscalablebf.h"
//#include "tdb.h"
#include "taosdef.h"
#include "tmsg.h"
#include "tcommon.h"
#ifdef __cplusplus
extern "C" {
#endif
#define TIMEWINDOW_RANGE_CONTAINED 1
#define TIMEWINDOW_RANGE_EXTERNAL 2
#define CACHESCAN_RETRIEVE_TYPE_ALL 0x1
#define CACHESCAN_RETRIEVE_TYPE_SINGLE 0x2
#define CACHESCAN_RETRIEVE_LAST_ROW 0x4
#define CACHESCAN_RETRIEVE_LAST 0x8
typedef struct SMeta SMeta;
typedef struct SMetaEntry {
int64_t version;
int8_t type;
int8_t flags; // TODO: need refactor?
tb_uid_t uid;
char * name;
union {
struct {
SSchemaWrapper schemaRow;
SSchemaWrapper schemaTag;
SRSmaParam rsmaParam;
} stbEntry;
struct {
int64_t ctime;
int32_t ttlDays;
int32_t commentLen;
char * comment;
tb_uid_t suid;
uint8_t *pTags;
} ctbEntry;
struct {
int64_t ctime;
int32_t ttlDays;
int32_t commentLen;
char * comment;
int32_t ncid; // next column id
SSchemaWrapper schemaRow;
} ntbEntry;
struct {
STSma *tsma;
} smaEntry;
};
uint8_t *pBuf;
} SMetaEntry;
// int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables);
// int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
// SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
// int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
// SArray *pTableUids);
// void *tsdbCacherowsReaderClose(void *pReader);
// int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
// int32_t tsdbReaderOpen(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
// SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables);
// int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
// void tsdbReaderSetId(STsdbReader *pReader, const char *idstr);
// void tsdbReaderClose(STsdbReader *pReader);
// int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
// int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
// void tsdbReleaseDataBlock(STsdbReader *pReader);
// SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
// int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
// int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
// int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
// void *tsdbGetIdx(void *pMeta);
// void *tsdbGetIvtIdx(void *pMeta);
// uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader);
// void tsdbReaderSetCloseFlag(STsdbReader *pReader);
// int64_t tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr);
// int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
// int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
// int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list);
// void *vnodeGetIdx(void *pVnode);
// void *vnodeGetIvtIdx(void *pVnode);
// void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
// void metaReaderReleaseLock(SMetaReader *pReader);
// void metaReaderClear(SMetaReader *pReader);
// int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
// int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
// int metaGetTableEntryByName(SMetaReader *pReader, const char *name);
// int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList);
// int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList);
// const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
// int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
//
// int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
// int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
// int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
// bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
// int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
// bool *acquired);
// int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
// int32_t payloadLen, double selectivityRatio);
// int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
// tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
// int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
// int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
// int32_t payloadLen);
// tq
typedef struct SMetaTableInfo {
int64_t suid;
int64_t uid;
SSchemaWrapper *schema;
char tbName[TSDB_TABLE_NAME_LEN];
} SMetaTableInfo;
typedef struct SSnapContext {
SMeta * pMeta;
int64_t snapVersion;
struct TBC *pCur;
int64_t suid;
int8_t subType;
SHashObj * idVersion;
SHashObj * suidInfo;
SArray * idList;
int32_t index;
bool withMeta;
bool queryMeta; // true-get meta, false-get data
} SSnapContext;
typedef struct {
int64_t uid;
int64_t ctbNum;
} SMetaStbStats;
// void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
// int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
// int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
// int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
// bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid);
// bool tqCurrentBlockConsumed(const STqReader* pReader);
//
// int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id);
// bool tqNextBlockInWal(STqReader* pReader, const char* idstr);
// bool tqNextBlockImpl(STqReader *pReader, const char* idstr);
// int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
// SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx);
// int32_t setForSnapShot(SSnapContext *ctx, int64_t uid);
// int32_t destroySnapContext(SSnapContext *ctx);
// SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
// void metaCloseTbCursor(SMTbCursor *pTbCur);
// int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType);
// int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType);
#define META_READER_NOLOCK 0x1
/*-------------------------------------------------new api format---------------------------------------------------*/
// typedef int32_t (*__store_reader_(STsdbReader *pReader, const void *pTableList, int32_t num);
// typedef void (*tsdbReaderSetId(STsdbReader *pReader, const char *idstr);
// typedef void (*tsdbReaderClose(STsdbReader *pReader);
// typedef int32_t (*tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
// typedef int32_t (*tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
// typedef void (*tsdbReleaseDataBlock(STsdbReader *pReader);
// typedef SSDataBlock * (*tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
// typedef int32_t (*tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
// typedef int32_t (*tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
// typedef int64_t (*tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
// typedef void * (*tsdbGetIdx(void *pMeta);
// typedef void * (*tsdbGetIvtIdx(void *pMeta);
// typedef uint64_t (*tsdbGetReaderMaxVersion(STsdbReader *pReader);
// typedef void (*tsdbReaderSetCloseFlag(STsdbReader *pReader);
// typedef int64_t (*tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr);
typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *pCond, void *pTableList,
int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader,
const char *idstr, bool countOnly, SHashObj **pIgnoreTables);
typedef struct SStoreDataReaderFn {
__store_reader_open_fn_t storeReaderOpen;
void (*storeReaderClose)();
void (*setReaderId)(void *pReader, const char *pId);
void (*storeReaderSetTableList)();
int32_t (*storeReaderNextDataBlock)();
int32_t (*storeReaderRetrieveBlockSMA)();
SSDataBlock *(*storeReaderRetrieveDataBlock)();
void (*storeReaderReleaseDataBlock)();
void (*storeReaderResetStatus)();
void (*storeReaderGetDataBlockDistInfo)();
void (*storeReaderGetNumOfInMemRows)();
void (*storeReaderNotifyClosing)();
} SStoreDataReaderFn;
/**
* int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUids);
void *tsdbCacherowsReaderClose(void *pReader);
*/
typedef struct SStoreCachedDataReaderFn {
int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr);
void *(*closeReader)(void *pReader);
int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUidList);
void (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables);
} SStoreCachedDataReaderFn;
/*------------------------------------------------------------------------------------------------------------------*/
/*
*
void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList);
int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid);
bool tqCurrentBlockConsumed(const STqReader* pReader);
int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id);
bool tqNextBlockInWal(STqReader* pReader, const char* idstr);
bool tqNextBlockImpl(STqReader *pReader, const char* idstr);
int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char* idstr);
STqReader *tqReaderOpen(void *pVnode);
void tqCloseReader(STqReader *);
int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
SWalReader* tqGetWalReader(STqReader* pReader);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
*/
// todo rename
typedef struct SStoreTqReaderFn {
void *(*tqReaderOpen)();
void (*tqReaderClose)();
int32_t (*tqReaderSeek)();
int32_t (*tqRetrieveBlock)();
bool (*tqReaderNextBlockInWal)();
bool (*tqNextBlockImpl)(); // todo remove it
void (*tqReaderSetColIdList)();
int32_t (*tqReaderSetTargetTableList)();
int32_t (*tqReaderAddTables)();
int32_t (*tqReaderRemoveTables)();
bool (*tqReaderIsQueriedTable)();
bool (*tqReaderCurrentBlockConsumed)();
struct SWalReader *(*tqReaderGetWalReader)(); // todo remove it
void (*tqReaderRetrieveTaosXBlock)(); // todo remove it
int32_t (*tqReaderSetSubmitMsg)(); // todo remove it
void (*tqReaderNextBlockFilterOut)();
} SStoreTqReaderFn;
typedef struct SStoreSnapshotFn {
/*
int32_t getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx);
int32_t setForSnapShot(SSnapContext *ctx, int64_t uid);
int32_t destroySnapContext(SSnapContext *ctx);
*/
int32_t (*storeCreateSnapshot)();
void (*storeDestroySnapshot)();
SMetaTableInfo (*storeSSGetTableInfo)();
int32_t (*storeSSGetMetaInfo)();
} SStoreSnapshotFn;
/**
void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
void metaReaderReleaseLock(SMetaReader *pReader);
void metaReaderClear(SMetaReader *pReader);
int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
int metaGetTableEntryByName(SMetaReader *pReader, const char *name);
int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList);
const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid);
int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid);
int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
bool *acquired);
int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen, double selectivityRatio);
int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
int32_t payloadLen);
*/
typedef struct SMetaReader {
int32_t flags;
void * pMeta;
SDecoder coder;
SMetaEntry me;
void * pBuf;
int32_t szBuf;
struct SStorageAPI *storageAPI;
} SMetaReader;
typedef struct SStoreMetaReaderFn {
void (*initReader)(void *pReader, void *pMeta, int32_t flags);
void *(*clearReader)();
void (*readerReleaseLock)();
int32_t (*getTableEntryByUid)();
int32_t (*getTableEntryByName)();
int32_t (*readerGetEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid);
} SStoreMetaReaderFn;
typedef struct SStoreMetaFn {
/*
SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
void metaCloseTbCursor(SMTbCursor *pTbCur);
int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType);
int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType);
*/
void *(*openMetaCursor)();
void (*closeMetaCursor)();
int32_t (*cursorNext)();
void (*cursorPrev)();
int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList);
int32_t (*getTableTagsByUid)();
const char *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it
int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid);
int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType);
int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName);
bool (*isTableExisted)(void *pVnode, uint64_t uid);
/**
* int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen, double selectivityRatio);
int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid);
tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList);
int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
int32_t payloadLen);
*/
void (*getCachedTableList)();
void (*putTableListIntoCache)();
/**
*
*/
void *(*storeGetIndexInfo)();
void *(*storeGetInvertIndex)();
void (*storeGetChildTableList)(); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
int32_t (*storeGetTableList)(); // vnodeGetStbIdList & vnodeGetAllTableList
void *storeGetVersionRange;
void *storeGetLastTimestamp;
int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema
// db name, vgId, numOfTables, numOfSTables
void (*storeGetNumOfChildTables)(); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo);
void (*storeGetBasicInfo)(); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta);
int64_t (*getNumOfRowsInMem)();
/**
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list);
*/
} SStoreMetaFn;
typedef struct STdbState {
void* rocksdb;
void** pHandle;
void* writeOpts;
void* readOpts;
void** cfOpts;
void* dbOpt;
struct SStreamTask* pOwner;
void* param;
void* env;
SListNode* pComparNode;
void* pBackendHandle;
char idstr[64];
void* compactFactory;
void* db;
void* pStateDb;
void* pFuncStateDb;
void* pFillStateDb; // todo refactor
void* pSessionStateDb;
void* pParNameDb;
void* pParTagDb;
void* txn;
} STdbState;
// incremental state storage
typedef struct {
STdbState* pTdbState;
struct SStreamFileState* pFileState;
int32_t number;
SSHashObj* parNameMap;
int64_t checkPointId;
int32_t taskId;
int64_t streamId;
} SStreamState;
typedef struct SUpdateInfo {
SArray *pTsBuckets;
uint64_t numBuckets;
SArray *pTsSBFs;
uint64_t numSBFs;
int64_t interval;
int64_t watermark;
TSKEY minTS;
SScalableBf *pCloseWinSBF;
SHashObj *pMap;
uint64_t maxDataVersion;
} SUpdateInfo;
typedef struct {
void* iter;
void* snapshot;
void* readOpt;
void* db;
// rocksdb_iterator_t* iter;
// rocksdb_snapshot_t* snapshot;
// rocksdb_readoptions_t* readOpt;
// rocksdb_t* db;
void* pCur;
int64_t number;
} SStreamStateCur;
typedef TSKEY (*GetTsFun)(void*);
typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateReleaseBuf)(SStreamState* pState, const SWinKey* key, void* pVal);
void (*streamStateFreeVal)(void* val);
int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal);
int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateClear)(SStreamState* pState);
void (*streamStateSetNumber)(SStreamState* pState, int32_t number);
int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateCurNext)(SStreamState* pState, void* pCur);
int32_t (*streamStateCurPrev)(SStreamState* pState, void* pCur);
void* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key);
void* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key);
void* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key);
void* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key);
void (*streamStateFreeCur)(void* pCur);
int32_t (*streamStateGetGroupKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t (*streamStateGetKVByCur)(void* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen);
int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key);
int32_t (*streamStateSessionClear)(SStreamState* pState);
int32_t (*streamStateSessionGetKVByCur)(void* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionGetKeyByRange)(void* pState, const SSessionKey* range, SSessionKey* curKey);
void* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark);
TSKEY (*updateInfoFillBlockData)(void *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol);
bool (*updateInfoIsUpdated)(void *pInfo, uint64_t tableId, TSKEY ts);
bool (*updateInfoIsTableInserted)(void *pInfo, int64_t tbUid);
void (*updateInfoDestroy)(void *pInfo);
SUpdateInfo* (*updateInfoInitP)(SInterval *pInterval, int64_t watermark);
void (*updateInfoAddCloseWindowSBF)(void *pInfo);
void (*updateInfoDestoryColseWinSBF)(void *pInfo);
int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const void *pInfo);
int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, void *pInfo);
void* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);
void* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key);
void* (*streamStateSessionSeekKeyCurrentNext)(SStreamState* pState, const SSessionKey* key);
void* (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp,
void* pFile, TSKEY delMark);
void (*streamFileStateDestroy)(void* pFileState);
void (*streamFileStateClear)(void* pFileState);
bool (*needClearDiskBuff)(void* pFileState);
SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages);
void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState);
int32_t (*streamStateCommit)(SStreamState* pState);
void (*streamStateDestroy)(SStreamState* pState, bool remove);
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
} SStateStore;
typedef struct SStorageAPI {
SStoreMetaFn metaFn; // todo: refactor
SStoreDataReaderFn storeReader;
SStoreMetaReaderFn metaReaderFn;
SStoreCachedDataReaderFn cacheFn;
SStoreSnapshotFn snapshotFn;
SStoreTqReaderFn tqReaderFn;
SStateStore stateStore;
} SStorageAPI;
typedef struct SMTbCursor {
struct TBC *pDbc;
void * pKey;
void * pVal;
int32_t kLen;
int32_t vLen;
SMetaReader mr;
} SMTbCursor;
typedef struct SRowBuffPos {
void* pRowBuff;
void* pKey;
bool beFlushed;
bool beUsed;
} SRowBuffPos;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_STORAGEAPI_H

31
include/libs/index/index.h
View File

@ -21,6 +21,7 @@
#include "taoserror.h" #include "taoserror.h"
#include "tarray.h" #include "tarray.h"
#include "tglobal.h" #include "tglobal.h"
#include "tsdstorage.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -231,6 +232,36 @@ void indexInit(int32_t threads);
*/ */
void indexCleanup(); void indexCleanup();
/**
* the underlying storage module must implement this API to employ the index functions.
* @param pMeta
* @param param
* @param results
* @return
*/
typedef struct SMetaFltParam {
uint64_t suid;
int16_t cid;
int16_t type;
void *val;
bool reverse;
bool equal;
int (*filterFunc)(void *a, void *b, int16_t type);
} SMetaFltParam;
typedef struct SStoreAPI {
int32_t (*metaFilterTableIds)();
int32_t (*metaFilterCreateTime)();
int32_t (*metaFilterTableName)();
int32_t (*metaFilterTtl)();
} SStoreAPI;
//int32_t metaFilterTableIds(void *pMeta, SMetaFltParam *param, SArray *results);
//int32_t metaFilterCreateTime(void *pMeta, SMetaFltParam *parm, SArray *pUids);
//int32_t metaFilterTableName(void *pMeta, SMetaFltParam *param, SArray *pUids);
//int32_t metaFilterTtl(void *pMeta, SMetaFltParam *param, SArray *pUids);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

82
include/libs/stream/streamState.h
View File

@ -27,47 +27,37 @@
extern "C" { extern "C" {
#endif #endif
#include "storageapi.h"
// void* streamBackendInit(const char* path); // void* streamBackendInit(const char* path);
// void streamBackendCleanup(void* arg); // void streamBackendCleanup(void* arg);
// SListNode* streamBackendAddCompare(void* backend, void* arg); // SListNode* streamBackendAddCompare(void* backend, void* arg);
// void streamBackendDelCompare(void* backend, void* arg); // void streamBackendDelCompare(void* backend, void* arg);
typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
typedef struct STdbState { //typedef struct STdbState {
rocksdb_t* rocksdb; // rocksdb_t* rocksdb;
rocksdb_column_family_handle_t** pHandle; // rocksdb_column_family_handle_t** pHandle;
rocksdb_writeoptions_t* writeOpts; // rocksdb_writeoptions_t* writeOpts;
rocksdb_readoptions_t* readOpts; // rocksdb_readoptions_t* readOpts;
rocksdb_options_t** cfOpts; // rocksdb_options_t** cfOpts;
rocksdb_options_t* dbOpt; // rocksdb_options_t* dbOpt;
struct SStreamTask* pOwner; // struct SStreamTask* pOwner;
void* param; // void* param;
void* env; // void* env;
SListNode* pComparNode; // SListNode* pComparNode;
void* pBackendHandle; // void* pBackendHandle;
char idstr[64]; // char idstr[64];
void* compactFactory; // void* compactFactory;
//
TDB* db; // TDB* db;
TTB* pStateDb; // TTB* pStateDb;
TTB* pFuncStateDb; // TTB* pFuncStateDb;
TTB* pFillStateDb; // todo refactor // TTB* pFillStateDb; // todo refactor
TTB* pSessionStateDb; // TTB* pSessionStateDb;
TTB* pParNameDb; // TTB* pParNameDb;
TTB* pParTagDb; // TTB* pParTagDb;
TXN* txn; // TXN* txn;
} STdbState; //} STdbState;
// incremental state storage
typedef struct {
STdbState* pTdbState;
SStreamFileState* pFileState;
int32_t number;
SSHashObj* parNameMap;
int64_t checkPointId;
int32_t taskId;
int64_t streamId;
} SStreamState;
SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages);
void streamStateClose(SStreamState* pState, bool remove); void streamStateClose(SStreamState* pState, bool remove);
@ -76,15 +66,15 @@ int32_t streamStateCommit(SStreamState* pState);
void streamStateDestroy(SStreamState* pState, bool remove); void streamStateDestroy(SStreamState* pState, bool remove);
int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
typedef struct { //typedef struct {
rocksdb_iterator_t* iter; // rocksdb_iterator_t* iter;
rocksdb_snapshot_t* snapshot; // rocksdb_snapshot_t* snapshot;
rocksdb_readoptions_t* readOpt; // rocksdb_readoptions_t* readOpt;
rocksdb_t* db; // rocksdb_t* db;
//
TBC* pCur; // TBC* pCur;
int64_t number; // int64_t number;
} SStreamStateCur; //} SStreamStateCur;
int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
@ -119,7 +109,7 @@ int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal);
void streamFreeVal(void* val); void streamStateFreeVal(void* val);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key);

9
include/libs/stream/tstreamFileState.h
View File

@ -21,23 +21,16 @@
#include "tarray.h" #include "tarray.h"
#include "tdef.h" #include "tdef.h"
#include "tlist.h" #include "tlist.h"
#include "storageapi.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct SStreamFileState SStreamFileState; typedef struct SStreamFileState SStreamFileState;
typedef struct SRowBuffPos {
void* pRowBuff;
void* pKey;
bool beFlushed;
bool beUsed;
} SRowBuffPos;
typedef SList SStreamSnapshot; typedef SList SStreamSnapshot;
typedef TSKEY (*GetTsFun)(void*);
SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
GetTsFun fp, void* pFile, TSKEY delMark); GetTsFun fp, void* pFile, TSKEY delMark);
void streamFileStateDestroy(SStreamFileState* pFileState); void streamFileStateDestroy(SStreamFileState* pFileState);

26
include/libs/stream/tstreamUpdate.h
View File

@ -19,7 +19,7 @@
#include "tarray.h" #include "tarray.h"
#include "tcommon.h" #include "tcommon.h"
#include "tmsg.h" #include "tmsg.h"
#include "tscalablebf.h" #include "storageapi.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -30,18 +30,18 @@ typedef struct SUpdateKey {
TSKEY ts; TSKEY ts;
} SUpdateKey; } SUpdateKey;
typedef struct SUpdateInfo { //typedef struct SUpdateInfo {
SArray *pTsBuckets; // SArray *pTsBuckets;
uint64_t numBuckets; // uint64_t numBuckets;
SArray *pTsSBFs; // SArray *pTsSBFs;
uint64_t numSBFs; // uint64_t numSBFs;
int64_t interval; // int64_t interval;
int64_t watermark; // int64_t watermark;
TSKEY minTS; // TSKEY minTS;
SScalableBf *pCloseWinSBF; // SScalableBf *pCloseWinSBF;
SHashObj *pMap; // SHashObj *pMap;
uint64_t maxDataVersion; // uint64_t maxDataVersion;
} SUpdateInfo; //} SUpdateInfo;
SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark); SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark); SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);

2
source/dnode/qnode/CMakeLists.txt
View File

@ -14,4 +14,6 @@ target_link_libraries(
PRIVATE qworker PRIVATE qworker
PRIVATE qcom PRIVATE qcom
PRIVATE executor PRIVATE executor
PRIVATE tdb
PRIVATE wal
) )

1
source/dnode/qnode/src/qnode.c
View File

@ -14,7 +14,6 @@
*/ */
#include "executor.h" #include "executor.h"
#include "libs/function/function.h"
#include "qndInt.h" #include "qndInt.h"
#include "query.h" #include "query.h"
#include "qworker.h" #include "qworker.h"

1
source/dnode/snode/CMakeLists.txt
View File

@ -14,4 +14,5 @@ target_link_libraries(
PRIVATE util PRIVATE util
PRIVATE qcom PRIVATE qcom
PRIVATE stream PRIVATE stream
PRIVATE wal
) )

186
source/dnode/vnode/inc/vnode.h
View File

@ -33,6 +33,7 @@
#include "trow.h" #include "trow.h"
#include "tdb.h" #include "tdb.h"
#include "storageapi.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -132,28 +133,28 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in
int64_t metaGetTbNum(SMeta *pMeta); int64_t metaGetTbNum(SMeta *pMeta);
int64_t metaGetNtbNum(SMeta *pMeta); int64_t metaGetNtbNum(SMeta *pMeta);
typedef struct { //typedef struct {
int64_t uid; // int64_t uid;
int64_t ctbNum; // int64_t ctbNum;
} SMetaStbStats; //} SMetaStbStats;
int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo);
typedef struct SMetaFltParam { //typedef struct SMetaFltParam {
tb_uid_t suid; // tb_uid_t suid;
int16_t cid; // int16_t cid;
int16_t type; // int16_t type;
void *val; // void *val;
bool reverse; // bool reverse;
bool equal; // bool equal;
int (*filterFunc)(void *a, void *b, int16_t type); // int (*filterFunc)(void *a, void *b, int16_t type);
//
} SMetaFltParam; //} SMetaFltParam;
// TODO, refactor later // TODO, refactor later
int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results); //int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *results);
int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids); //int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *parm, SArray *pUids);
int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); //int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids);
int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids); //int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids);
#if 1 // refact APIs below (TODO) #if 1 // refact APIs below (TODO)
typedef SVCreateTbReq STbCfg; typedef SVCreateTbReq STbCfg;
@ -213,32 +214,32 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode);
size_t tsdbCacheGetUsage(SVnode *pVnode); size_t tsdbCacheGetUsage(SVnode *pVnode);
int32_t tsdbCacheGetElems(SVnode *pVnode); int32_t tsdbCacheGetElems(SVnode *pVnode);
// tq //// tq
typedef struct SMetaTableInfo { //typedef struct SMetaTableInfo {
int64_t suid; // int64_t suid;
int64_t uid; // int64_t uid;
SSchemaWrapper *schema; // SSchemaWrapper *schema;
char tbName[TSDB_TABLE_NAME_LEN]; // char tbName[TSDB_TABLE_NAME_LEN];
} SMetaTableInfo; //} SMetaTableInfo;
typedef struct SIdInfo { typedef struct SIdInfo {
int64_t version; int64_t version;
int32_t index; int32_t index;
} SIdInfo; } SIdInfo;
typedef struct SSnapContext { //typedef struct SSnapContext {
SMeta *pMeta; // SMeta *pMeta;
int64_t snapVersion; // int64_t snapVersion;
TBC *pCur; // TBC *pCur;
int64_t suid; // int64_t suid;
int8_t subType; // int8_t subType;
SHashObj *idVersion; // SHashObj *idVersion;
SHashObj *suidInfo; // SHashObj *suidInfo;
SArray *idList; // SArray *idList;
int32_t index; // int32_t index;
bool withMeta; // bool withMeta;
bool queryMeta; // true-get meta, false-get data // bool queryMeta; // true-get meta, false-get data
} SSnapContext; //} SSnapContext;
typedef struct STqReader { typedef struct STqReader {
SPackedData msg; SPackedData msg;
@ -271,7 +272,7 @@ bool tqNextBlockImpl(STqReader *pReader, const char* idstr);
int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id); int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id);
int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(STqReader *pReader, const char* idstr); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock** pRes, const char* idstr);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg);
@ -350,67 +351,62 @@ struct SVnodeCfg {
int32_t tsdbPageSize; int32_t tsdbPageSize;
}; };
typedef struct {
uint64_t uid;
uint64_t groupId;
} STableKeyInfo;
#define TABLE_ROLLUP_ON ((int8_t)0x1) #define TABLE_ROLLUP_ON ((int8_t)0x1)
#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) #define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0)
#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) #define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON)
struct SMetaEntry { //struct SMetaEntry {
int64_t version; // int64_t version;
int8_t type; // int8_t type;
int8_t flags; // TODO: need refactor? // int8_t flags; // TODO: need refactor?
tb_uid_t uid; // tb_uid_t uid;
char *name; // char *name;
union { // union {
struct { // struct {
SSchemaWrapper schemaRow; // SSchemaWrapper schemaRow;
SSchemaWrapper schemaTag; // SSchemaWrapper schemaTag;
SRSmaParam rsmaParam; // SRSmaParam rsmaParam;
} stbEntry; // } stbEntry;
struct { // struct {
int64_t ctime; // int64_t ctime;
int32_t ttlDays; // int32_t ttlDays;
int32_t commentLen; // int32_t commentLen;
char *comment; // char *comment;
tb_uid_t suid; // tb_uid_t suid;
uint8_t *pTags; // uint8_t *pTags;
} ctbEntry; // } ctbEntry;
struct { // struct {
int64_t ctime; // int64_t ctime;
int32_t ttlDays; // int32_t ttlDays;
int32_t commentLen; // int32_t commentLen;
char *comment; // char *comment;
int32_t ncid; // next column id // int32_t ncid; // next column id
SSchemaWrapper schemaRow; // SSchemaWrapper schemaRow;
} ntbEntry; // } ntbEntry;
struct { // struct {
STSma *tsma; // STSma *tsma;
} smaEntry; // } smaEntry;
}; // };
//
// uint8_t *pBuf;
//};
uint8_t *pBuf; //struct SMetaReader {
}; // int32_t flags;
// SMeta *pMeta;
// SDecoder coder;
// SMetaEntry me;
// void *pBuf;
// int32_t szBuf;
//};
struct SMetaReader { //struct SMTbCursor {
int32_t flags; // TBC *pDbc;
SMeta *pMeta; // void *pKey;
SDecoder coder; // void *pVal;
SMetaEntry me; // int32_t kLen;
void *pBuf; // int32_t vLen;
int32_t szBuf; // SMetaReader mr;
}; //};
struct SMTbCursor {
TBC *pDbc;
void *pKey;
void *pVal;
int32_t kLen;
int32_t vLen;
SMetaReader mr;
};
#ifdef __cplusplus #ifdef __cplusplus
} }

3
source/dnode/vnode/src/inc/tsdb.h
View File

@ -844,9 +844,6 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode);
// int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); // int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
// ========== inline functions ========== // ========== inline functions ==========

12
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -249,7 +249,7 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) {
SMetaReader *pReader = &mr; SMetaReader *pReader = &mr;
// query name.idx // query name.idx
if (tdbTbGet(pReader->pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { if (tdbTbGet(((SMeta*)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
metaReaderClear(&mr); metaReaderClear(&mr);
return -1; return -1;
@ -293,9 +293,9 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
metaReaderInit(&pTbCur->mr, pMeta, 0); metaReaderInit(&pTbCur->mr, pMeta, 0);
tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); tdbTbcOpen(pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL);
tdbTbcMoveToFirst(pTbCur->pDbc); tdbTbcMoveToFirst((TBC *)pTbCur->pDbc);
return pTbCur; return pTbCur;
} }
@ -306,7 +306,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) {
tdbFree(pTbCur->pVal); tdbFree(pTbCur->pVal);
metaReaderClear(&pTbCur->mr); metaReaderClear(&pTbCur->mr);
if (pTbCur->pDbc) { if (pTbCur->pDbc) {
tdbTbcClose(pTbCur->pDbc); tdbTbcClose((TBC *)pTbCur->pDbc);
} }
taosMemoryFree(pTbCur); taosMemoryFree(pTbCur);
} }
@ -318,7 +318,7 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) {
STbCfg tbCfg; STbCfg tbCfg;
for (;;) { for (;;) {
ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); ret = tdbTbcNext((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -342,7 +342,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) {
STbCfg tbCfg; STbCfg tbCfg;
for (;;) { for (;;) {
ret = tdbTbcPrev(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); ret = tdbTbcPrev((TBC *)pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }

30
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -222,29 +222,29 @@ static void destroySTableInfoForChildTable(void* data) {
} }
static void MoveToSnapShotVersion(SSnapContext* ctx) { static void MoveToSnapShotVersion(SSnapContext* ctx) {
tdbTbcClose(ctx->pCur); tdbTbcClose((TBC*)ctx->pCur);
tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL);
STbDbKey key = {.version = ctx->snapVersion, .uid = INT64_MAX}; STbDbKey key = {.version = ctx->snapVersion, .uid = INT64_MAX};
int c = 0; int c = 0;
tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c); tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c);
if (c < 0) { if (c < 0) {
tdbTbcMoveToPrev(ctx->pCur); tdbTbcMoveToPrev((TBC*)ctx->pCur);
} }
} }
static int32_t MoveToPosition(SSnapContext* ctx, int64_t ver, int64_t uid) { static int32_t MoveToPosition(SSnapContext* ctx, int64_t ver, int64_t uid) {
tdbTbcClose(ctx->pCur); tdbTbcClose((TBC*)ctx->pCur);
tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL);
STbDbKey key = {.version = ver, .uid = uid}; STbDbKey key = {.version = ver, .uid = uid};
int c = 0; int c = 0;
tdbTbcMoveTo(ctx->pCur, &key, sizeof(key), &c); tdbTbcMoveTo((TBC*)ctx->pCur, &key, sizeof(key), &c);
return c; return c;
} }
static void MoveToFirst(SSnapContext* ctx) { static void MoveToFirst(SSnapContext* ctx) {
tdbTbcClose(ctx->pCur); tdbTbcClose((TBC*)ctx->pCur);
tdbTbcOpen(ctx->pMeta->pTbDb, &ctx->pCur, NULL); tdbTbcOpen(ctx->pMeta->pTbDb, (TBC**)&ctx->pCur, NULL);
tdbTbcMoveToFirst(ctx->pCur); tdbTbcMoveToFirst((TBC*)ctx->pCur);
} }
static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) { static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) {
@ -291,7 +291,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
metaDebug("tmqsnap init snapVersion:%" PRIi64, ctx->snapVersion); metaDebug("tmqsnap init snapVersion:%" PRIi64, ctx->snapVersion);
MoveToFirst(ctx); MoveToFirst(ctx);
while (1) { while (1) {
int32_t ret = tdbTbcNext(ctx->pCur, &pKey, &kLen, &pVal, &vLen); int32_t ret = tdbTbcNext((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break; if (ret < 0) break;
STbDbKey* tmp = (STbDbKey*)pKey; STbDbKey* tmp = (STbDbKey*)pKey;
if (tmp->version > ctx->snapVersion) break; if (tmp->version > ctx->snapVersion) break;
@ -329,7 +329,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
MoveToSnapShotVersion(ctx); MoveToSnapShotVersion(ctx);
while (1) { while (1) {
int32_t ret = tdbTbcPrev(ctx->pCur, &pKey, &kLen, &pVal, &vLen); int32_t ret = tdbTbcPrev((TBC*)ctx->pCur, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break; if (ret < 0) break;
STbDbKey* tmp = (STbDbKey*)pKey; STbDbKey* tmp = (STbDbKey*)pKey;
@ -378,7 +378,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t
} }
int32_t destroySnapContext(SSnapContext* ctx) { int32_t destroySnapContext(SSnapContext* ctx) {
tdbTbcClose(ctx->pCur); tdbTbcClose((TBC*)ctx->pCur);
taosArrayDestroy(ctx->idList); taosArrayDestroy(ctx->idList);
taosHashCleanup(ctx->idVersion); taosHashCleanup(ctx->idVersion);
taosHashCleanup(ctx->suidInfo); taosHashCleanup(ctx->suidInfo);
@ -496,7 +496,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in
metaDebug("tmqsnap get meta not exist uid:%" PRIi64 " version:%" PRIi64, *uid, idInfo->version); metaDebug("tmqsnap get meta not exist uid:%" PRIi64 " version:%" PRIi64, *uid, idInfo->version);
} }
tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
SDecoder dc = {0}; SDecoder dc = {0};
SMetaEntry me = {0}; SMetaEntry me = {0};
tDecoderInit(&dc, pVal, vLen); tDecoderInit(&dc, pVal, vLen);
@ -622,7 +622,7 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) {
metaDebug("tmqsnap getUidfromSnapShot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version); metaDebug("tmqsnap getUidfromSnapShot not exist uid:%" PRIi64 " version:%" PRIi64, *uidTmp, idInfo->version);
continue; continue;
} }
tdbTbcGet(ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen); tdbTbcGet((TBC*)ctx->pCur, (const void**)&pKey, &kLen, (const void**)&pVal, &vLen);
SDecoder dc = {0}; SDecoder dc = {0};
SMetaEntry me = {0}; SMetaEntry me = {0};
tDecoderInit(&dc, pVal, vLen); tDecoderInit(&dc, pVal, vLen);

25
source/dnode/vnode/src/tq/tqRead.c
View File

@ -394,8 +394,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) {
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if (pReader->tbIdHash == NULL) { if (pReader->tbIdHash == NULL) {
int32_t code = tqRetrieveDataBlock(pReader, NULL); SSDataBlock* pRes = NULL;
if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL);
if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
return true; return true;
} }
} }
@ -404,8 +405,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) {
if (ret != NULL) { if (ret != NULL) {
tqDebug("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver); tqDebug("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver);
int32_t code = tqRetrieveDataBlock(pReader, NULL); SSDataBlock* pRes = NULL;
if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL);
if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
return true; return true;
} }
} else { } else {
@ -440,6 +442,11 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i
return 0; return 0;
} }
SWalReader* tqGetWalReader(STqReader* pReader) {
return pReader->pWalReader;
}
bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { bool tqNextBlockImpl(STqReader* pReader, const char* idstr) {
if (pReader->msg.msgStr == NULL) { if (pReader->msg.msgStr == NULL) {
return false; return false;
@ -592,7 +599,7 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol
return code; return code;
} }
int32_t tqRetrieveDataBlock(STqReader* pReader, const char* id) { int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) {
tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++);
@ -1024,6 +1031,14 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) {
return 0; return 0;
} }
bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid) {
return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t));
}
bool tqCurrentBlockConsumed(const STqReader* pReader) {
return pReader->msg.msgStr == NULL;
}
int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);

2
source/libs/executor/CMakeLists.txt
View File

@ -3,7 +3,7 @@ aux_source_directory(src EXECUTOR_SRC)
add_library(executor STATIC ${EXECUTOR_SRC}) add_library(executor STATIC ${EXECUTOR_SRC})
target_link_libraries(executor target_link_libraries(executor
PRIVATE os util common function parser planner qcom vnode scalar nodes index PRIVATE os util common function parser planner qcom scalar nodes index wal tdb
) )
target_include_directories( target_include_directories(

4
source/libs/executor/inc/dataSinkInt.h
View File

@ -22,13 +22,15 @@ extern "C" {
#include "dataSinkMgt.h" #include "dataSinkMgt.h"
#include "plannodes.h" #include "plannodes.h"
#include "storageapi.h"
#include "tcommon.h" #include "tcommon.h"
struct SDataSink; struct SDataSink;
struct SDataSinkHandle; struct SDataSinkHandle;
typedef struct SDataSinkManager { typedef struct SDataSinkManager {
SDataSinkMgtCfg cfg; SDataSinkMgtCfg cfg;
SStorageAPI storeFn;
} SDataSinkManager; } SDataSinkManager;
typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue);

12
source/libs/executor/inc/executil.h
View File

@ -12,17 +12,17 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef TDENGINE_QUERYUTIL_H #ifndef TDENGINE_EXECUTIL_H
#define TDENGINE_QUERYUTIL_H #define TDENGINE_EXECUTIL_H
#include "executor.h" #include "executor.h"
#include "function.h" #include "function.h"
#include "nodes.h" #include "nodes.h"
#include "plannodes.h" #include "plannodes.h"
#include "storageapi.h"
#include "tcommon.h" #include "tcommon.h"
#include "tpagedbuf.h" #include "tpagedbuf.h"
#include "tsimplehash.h" #include "tsimplehash.h"
#include "vnode.h"
#define T_LONG_JMP(_obj, _c) \ #define T_LONG_JMP(_obj, _c) \
do { \ do { \
@ -154,7 +154,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode); SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode);
EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId, SStorageAPI* pAPI);
size_t getTableTagsBufLen(const SNodeList* pGroups); size_t getTableTagsBufLen(const SNodeList* pGroups);
SArray* createSortInfo(SNodeList* pNodeList); SArray* createSortInfo(SNodeList* pNodeList);
@ -178,8 +178,8 @@ void cleanupQueryTableDataCond(SQueryTableDataCond* pCond);
int32_t convertFillType(int32_t mode); int32_t convertFillType(int32_t mode);
int32_t resultrowComparAsc(const void* p1, const void* p2); int32_t resultrowComparAsc(const void* p1, const void* p2);
int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI);
void printDataBlock(SSDataBlock* pBlock, const char* flag); void printDataBlock(SSDataBlock* pBlock, const char* flag);
#endif // TDENGINE_QUERYUTIL_H #endif // TDENGINE_EXECUTIL_H

51
source/libs/executor/inc/executorInt.h
View File

@ -38,13 +38,15 @@ extern "C" {
#include "tlockfree.h" #include "tlockfree.h"
#include "tmsg.h" #include "tmsg.h"
#include "tpagedbuf.h" #include "tpagedbuf.h"
#include "tstream.h" //#include "tstream.h"
#include "tstreamUpdate.h" //#include "tstreamUpdate.h"
#include "tlrucache.h"
#include "vnode.h"
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
typedef struct STsdbReader STsdbReader;
typedef struct STqReader STqReader;
#define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) #define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0)
#define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) #define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN)
#define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) #define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0)
@ -206,6 +208,7 @@ typedef struct STableScanBase {
SLimitInfo limitInfo; SLimitInfo limitInfo;
// there are more than one table list exists in one task, if only one vnode exists. // there are more than one table list exists in one task, if only one vnode exists.
STableListInfo* pTableListInfo; STableListInfo* pTableListInfo;
SStoreDataReaderFn readerAPI;
} STableScanBase; } STableScanBase;
typedef struct STableScanInfo { typedef struct STableScanInfo {
@ -221,6 +224,7 @@ typedef struct STableScanInfo {
int8_t assignBlockUid; int8_t assignBlockUid;
bool hasGroupByTag; bool hasGroupByTag;
bool countOnly; bool countOnly;
SStoreDataReaderFn readerAPI;
} STableScanInfo; } STableScanInfo;
typedef struct STableMergeScanInfo { typedef struct STableMergeScanInfo {
@ -280,6 +284,7 @@ typedef struct SStreamAggSupporter {
int32_t stateKeySize; int32_t stateKeySize;
int16_t stateKeyType; int16_t stateKeyType;
SDiskbasedBuf* pResultBuf; SDiskbasedBuf* pResultBuf;
SStateStore stateStore;
} SStreamAggSupporter; } SStreamAggSupporter;
typedef struct SWindowSupporter { typedef struct SWindowSupporter {
@ -335,7 +340,7 @@ typedef struct SStreamScanInfo {
STqReader* tqReader; STqReader* tqReader;
uint64_t groupId; uint64_t groupId;
SUpdateInfo* pUpdateInfo; struct SUpdateInfo* pUpdateInfo;
EStreamScanMode scanMode; EStreamScanMode scanMode;
struct SOperatorInfo* pStreamScanOp; struct SOperatorInfo* pStreamScanOp;
@ -366,15 +371,18 @@ typedef struct SStreamScanInfo {
SSDataBlock* pCreateTbRes; SSDataBlock* pCreateTbRes;
int8_t igCheckUpdate; int8_t igCheckUpdate;
int8_t igExpired; int8_t igExpired;
SStreamState* pState; void* pState; //void
SStoreTqReaderFn readerFn;
SStateStore stateStore;
} SStreamScanInfo; } SStreamScanInfo;
typedef struct { typedef struct {
SVnode* vnode; struct SVnode* vnode; // todo remove this
SSDataBlock pRes; // result SSDataBlock SSDataBlock pRes; // result SSDataBlock
STsdbReader* dataReader; STsdbReader* dataReader;
SSnapContext* sContext; struct SSnapContext* sContext;
STableListInfo* pTableListInfo; SStorageAPI* pAPI;
STableListInfo* pTableListInfo;
} SStreamRawScanInfo; } SStreamRawScanInfo;
typedef struct STableCountScanSupp { typedef struct STableCountScanSupp {
@ -441,12 +449,13 @@ typedef struct SStreamIntervalOperatorInfo {
bool isFinal; bool isFinal;
SArray* pChildren; SArray* pChildren;
int32_t numOfChild; int32_t numOfChild;
SStreamState* pState; SStreamState* pState; // void
SWinKey delKey; SWinKey delKey;
uint64_t numOfDatapack; uint64_t numOfDatapack;
SArray* pUpdated; SArray* pUpdated;
SSHashObj* pUpdatedMap; SSHashObj* pUpdatedMap;
int64_t dataVersion; int64_t dataVersion;
SStateStore statestore;
} SStreamIntervalOperatorInfo; } SStreamIntervalOperatorInfo;
typedef struct SDataGroupInfo { typedef struct SDataGroupInfo {
@ -543,6 +552,7 @@ typedef struct SStreamFillSupporter {
int32_t rowSize; int32_t rowSize;
SSHashObj* pResMap; SSHashObj* pResMap;
bool hasDelete; bool hasDelete;
SStorageAPI* pAPI;
} SStreamFillSupporter; } SStreamFillSupporter;
typedef struct SStreamFillOperatorInfo { typedef struct SStreamFillOperatorInfo {
@ -636,7 +646,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap);
bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool functionNeedToExecute(SqlFunctionCtx* pCtx);
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore);
void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid,
uint64_t* pGp, void* pTbName); uint64_t* pGp, void* pTbName);
uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);
@ -645,20 +655,17 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos
SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
bool groupbyTbname(SNodeList* pGroupList); bool groupbyTbname(SNodeList* pGroupList);
int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo); SGroupResInfo* pGroupResInfo);
int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size); int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI);
int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
SExprSupp* pSup, SGroupResInfo* pGroupResInfo); SExprSupp* pSup, SGroupResInfo* pGroupResInfo);
int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI);
SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup);
int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult);
int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order);
int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order,
int64_t* pData); int64_t* pData);
void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI);
SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag);
SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs); SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs);

2
source/libs/executor/inc/operator.h
View File

@ -156,7 +156,7 @@ void destroyOperator(SOperatorInfo* pOperator);
SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id); SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id);
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder);
int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr); int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI);
int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList); int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList);
#ifdef __cplusplus #ifdef __cplusplus

28
source/libs/executor/inc/querytask.h
View File

@ -56,20 +56,19 @@ typedef struct STaskStopInfo {
} STaskStopInfo; } STaskStopInfo;
typedef struct { typedef struct {
STqOffsetVal currentOffset; // for tmq STqOffsetVal currentOffset; // for tmq
SMqMetaRsp metaRsp; // for tmq fetching meta SMqMetaRsp metaRsp; // for tmq fetching meta
int64_t snapshotVer; int64_t snapshotVer;
// SPackedData submit; // todo remove it SSchemaWrapper* schema;
SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor int8_t recoverStep;
int8_t recoverStep; int8_t recoverScanFinished;
int8_t recoverScanFinished; SQueryTableDataCond tableCond;
SQueryTableDataCond tableCond; int64_t fillHistoryVer1;
int64_t fillHistoryVer1; int64_t fillHistoryVer2;
int64_t fillHistoryVer2; SStreamState* pState;
SStreamState* pState; int64_t dataVersion;
int64_t dataVersion; int64_t checkPointId;
int64_t checkPointId;
} SStreamTaskInfo; } SStreamTaskInfo;
struct SExecTaskInfo { struct SExecTaskInfo {
@ -92,6 +91,7 @@ struct SExecTaskInfo {
SArray* pResultBlockList; // result block list SArray* pResultBlockList; // result block list
STaskStopInfo stopInfo; STaskStopInfo stopInfo;
SRWLatch lock; // secure the access of STableListInfo SRWLatch lock; // secure the access of STableListInfo
SStorageAPI storageAPI;
}; };
void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst);

3
source/libs/executor/src/aggregateoperator.c
View File

@ -21,7 +21,7 @@
#include "tname.h" #include "tname.h"
#include "executorInt.h" #include "executorInt.h"
#include "index.h" #include "tsdstorage.h"
#include "operator.h" #include "operator.h"
#include "query.h" #include "query.h"
#include "querytask.h" #include "querytask.h"
@ -30,6 +30,7 @@
#include "tglobal.h" #include "tglobal.h"
#include "thash.h" #include "thash.h"
#include "ttypes.h" #include "ttypes.h"
#include "index.h"
typedef struct { typedef struct {
bool hasAgg; bool hasAgg;

22
source/libs/executor/src/cachescanoperator.c
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "function.h"
#include "os.h" #include "os.h"
#include "function.h"
#include "tname.h" #include "tname.h"
#include "tdatablock.h" #include "tdatablock.h"
@ -27,6 +27,8 @@
#include "thash.h" #include "thash.h"
#include "ttypes.h" #include "ttypes.h"
#include "storageapi.h"
typedef struct SCacheRowsScanInfo { typedef struct SCacheRowsScanInfo {
SSDataBlock* pRes; SSDataBlock* pRes;
SReadHandle readHandle; SReadHandle readHandle;
@ -102,9 +104,9 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
uint64_t suid = tableListGetSuid(pTableListInfo); uint64_t suid = tableListGetSuid(pTableListInfo);
code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables,
taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds, suid, taosArrayGetSize(pInfo->matchInfo.pList), pCidList, pInfo->pSlotIds,
&pInfo->pLastrowReader, pTaskInfo->id.str); suid, &pInfo->pLastrowReader, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
@ -172,7 +174,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
blockDataCleanup(pInfo->pBufferredRes); blockDataCleanup(pInfo->pBufferredRes);
taosArrayClear(pInfo->pUidList); taosArrayClear(pInfo->pUidList);
int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, int32_t code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds,
pInfo->pDstSlotIds, pInfo->pUidList); pInfo->pDstSlotIds, pInfo->pUidList);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
@ -239,7 +241,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
} }
if (NULL == pInfo->pLastrowReader) { if (NULL == pInfo->pLastrowReader) {
code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num, code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, num,
taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader, taosArrayGetSize(pInfo->matchInfo.pList), pInfo->pCidList, pInfo->pSlotIds, suid, &pInfo->pLastrowReader,
pTaskInfo->id.str); pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -248,12 +250,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
continue; continue;
} }
} else { } else {
tsdbReuseCacherowsReader(pInfo->pLastrowReader, pList, num); pInfo->readHandle.api.cacheFn.reuseReader(pInfo->pLastrowReader, pList, num);
} }
taosArrayClear(pInfo->pUidList); taosArrayClear(pInfo->pUidList);
code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds, code = pInfo->readHandle.api.cacheFn.retrieveRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pDstSlotIds,
pInfo->pUidList); pInfo->pUidList);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
@ -287,7 +289,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
} }
} }
pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader);
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
return NULL; return NULL;
} }
@ -305,7 +307,7 @@ void destroyCacheScanOperator(void* param) {
tableListDestroy(pInfo->pTableList); tableListDestroy(pInfo->pTableList);
if (pInfo->pLastrowReader != NULL) { if (pInfo->pLastrowReader != NULL) {
pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); pInfo->pLastrowReader = pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader);
} }
cleanupExprSupp(&pInfo->pseudoExprSup); cleanupExprSupp(&pInfo->pseudoExprSup);

4
source/libs/executor/src/dataInserter.c
View File

@ -17,6 +17,7 @@
#include "dataSinkMgt.h" #include "dataSinkMgt.h"
#include "executorInt.h" #include "executorInt.h"
#include "planner.h" #include "planner.h"
#include "storageapi.h"
#include "tcompression.h" #include "tcompression.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tglobal.h" #include "tglobal.h"
@ -428,8 +429,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
inserter->explain = pInserterNode->explain; inserter->explain = pInserterNode->explain;
int64_t suid = 0; int64_t suid = 0;
int32_t code = int32_t code = pManager->storeFn.metaFn.getTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid);
tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid);
if (code) { if (code) {
destroyDataSinker((SDataSinkHandle*)inserter); destroyDataSinker((SDataSinkHandle*)inserter);
taosMemoryFree(inserter); taosMemoryFree(inserter);

124
source/libs/executor/src/executil.c
View File

@ -27,6 +27,7 @@
#include "executorInt.h" #include "executorInt.h"
#include "querytask.h" #include "querytask.h"
#include "tcompression.h" #include "tcompression.h"
#include "storageapi.h"
typedef struct tagFilterAssist { typedef struct tagFilterAssist {
SHashObj* colHash; SHashObj* colHash;
@ -41,13 +42,13 @@ typedef enum {
} FilterCondType; } FilterCondType;
static FilterCondType checkTagCond(SNode* cond); static FilterCondType checkTagCond(SNode* cond);
static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SStorageAPI* pAPI);
static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond); static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond, SStorageAPI* pStoreAPI);
static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, static int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond,
SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr); SNode* pTagIndexCond, STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI);
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList,
void* metaHandle); void* pVnode, SStorageAPI* pStorageAPI);
static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; }
static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; }
@ -262,7 +263,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
STagVal tagVal = {0}; STagVal tagVal = {0};
tagVal.cid = pSColumnNode->colId; tagVal.cid = pSColumnNode->colId;
const char* p = metaGetTableTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); const char* p = mr->storageAPI->metaFn.extractTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal);
if (p == NULL) { if (p == NULL) {
res->node.resType.type = TSDB_DATA_TYPE_NULL; res->node.resType.type = TSDB_DATA_TYPE_NULL;
} else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) {
@ -301,14 +302,14 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified, SStorageAPI *pAPI) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, metaHandle, 0); pAPI->metaReaderFn.initReader(&mr, metaHandle, 0);
code = metaGetTableEntryByUidCache(&mr, info->uid); code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, info->uid);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
*pQualified = false; *pQualified = false;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -317,7 +318,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle,
SNode* pTagCondTmp = nodesCloneNode(pTagCond); SNode* pTagCondTmp = nodesCloneNode(pTagCond);
nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr); nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr);
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
SNode* pNew = NULL; SNode* pNew = NULL;
code = scalarCalculateConstants(pTagCondTmp, &pNew); code = scalarCalculateConstants(pTagCondTmp, &pNew);
@ -453,8 +454,8 @@ static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_C
taosMemoryFree(payload); taosMemoryFree(payload);
} }
int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInfo* pTableListInfo, uint8_t* digest,
int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo, uint8_t *digest) { SStorageAPI* pAPI) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SArray* pBlockList = NULL; SArray* pBlockList = NULL;
SSDataBlock* pResBlock = NULL; SSDataBlock* pResBlock = NULL;
@ -465,7 +466,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
int32_t rows = taosArrayGetSize(pTableListInfo->pTableList); int32_t rows = taosArrayGetSize(pTableListInfo->pTableList);
if (rows == 0) { if (rows == 0) {
return TDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
tagFilterAssist ctx = {0}; tagFilterAssist ctx = {0};
@ -494,8 +495,8 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
listNode->pNodeList = group; listNode->pNodeList = group;
genTbGroupDigest((SNode *)listNode, digest, &context); genTbGroupDigest((SNode *)listNode, digest, &context);
nodesFree(listNode); nodesFree(listNode);
metaGetCachedTbGroup(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); pAPI->metaFn.getCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList);
if (tableList) { if (tableList) {
taosArrayDestroy(pTableListInfo->pTableList); taosArrayDestroy(pTableListInfo->pTableList);
pTableListInfo->pTableList = tableList; pTableListInfo->pTableList = tableList;
@ -511,14 +512,13 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
taosArrayPush(pUidTagList, &info); taosArrayPush(pUidTagList, &info);
} }
// int64_t stt = taosGetTimestampUs(); code = pAPI->metaFn.getTableTags(pVnode, pTableListInfo->idInfo.suid, pUidTagList);
code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
int32_t numOfTables = taosArrayGetSize(pUidTagList); int32_t numOfTables = taosArrayGetSize(pUidTagList);
pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI);
if (pResBlock == NULL) { if (pResBlock == NULL) {
code = terrno; code = terrno;
goto end; goto end;
@ -632,7 +632,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
if (tsTagFilterCache) { if (tsTagFilterCache) {
tableList = taosArrayDup(pTableListInfo->pTableList, NULL); tableList = taosArrayDup(pTableListInfo->pTableList, NULL);
metaPutTbGroupToCache(metaHandle, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); pAPI->metaFn.putTableListIntoCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo));
} }
// int64_t st2 = taosGetTimestampUs(); // int64_t st2 = taosGetTimestampUs();
@ -734,12 +734,12 @@ static FilterCondType checkTagCond(SNode* cond) {
return FILTER_OTHER; return FILTER_OTHER;
} }
static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond) { static int32_t optimizeTbnameInCond(void* pVnode, int64_t suid, SArray* list, SNode* cond, SStorageAPI* pAPI) {
int32_t ret = -1; int32_t ret = -1;
int32_t ntype = nodeType(cond); int32_t ntype = nodeType(cond);
if (ntype == QUERY_NODE_OPERATOR) { if (ntype == QUERY_NODE_OPERATOR) {
ret = optimizeTbnameInCondImpl(metaHandle, list, cond); ret = optimizeTbnameInCondImpl(pVnode, list, cond, pAPI);
} }
if (ntype != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { if (ntype != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) {
@ -758,7 +758,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
SListCell* cell = pList->pHead; SListCell* cell = pList->pHead;
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
if (cell == NULL) break; if (cell == NULL) break;
if (optimizeTbnameInCondImpl(metaHandle, list, cell->pNode) == 0) { if (optimizeTbnameInCondImpl(pVnode, list, cell->pNode, pAPI) == 0) {
hasTbnameCond = true; hasTbnameCond = true;
break; break;
} }
@ -769,14 +769,14 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
taosArrayRemoveDuplicate(list, filterTableInfoCompare, NULL); taosArrayRemoveDuplicate(list, filterTableInfoCompare, NULL);
if (hasTbnameCond) { if (hasTbnameCond) {
ret = metaGetTableTagsByUids(metaHandle, suid, list); ret = pAPI->metaFn.getTableTagsByUid(pVnode, suid, list);
} }
return ret; return ret;
} }
// only return uid that does not contained in pExistedUidList // only return uid that does not contained in pExistedUidList
static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) { static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, SNode* pTagCond, SStorageAPI* pStoreAPI) {
if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) {
return -1; return -1;
} }
@ -813,9 +813,9 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis
char* name = taosArrayGetP(pTbList, i); char* name = taosArrayGetP(pTbList, i);
uint64_t uid = 0; uint64_t uid = 0;
if (metaGetTableUidByName(metaHandle, name, &uid) == 0) { if (pStoreAPI->metaFn.getTableUidByName(pVnode, name, &uid) == 0) {
ETableType tbType = TSDB_TABLE_MAX; ETableType tbType = TSDB_TABLE_MAX;
if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { if (pStoreAPI->metaFn.getTableTypeByName(pVnode, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) { if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) {
STUidTagInfo s = {.uid = uid, .name = name, .pTagVal = NULL}; STUidTagInfo s = {.uid = uid, .name = name, .pTagVal = NULL};
taosArrayPush(pExistedUidList, &s); taosArrayPush(pExistedUidList, &s);
@ -841,7 +841,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList,
void* metaHandle) { void* pVnode, SStorageAPI* pStorageAPI) {
SSDataBlock* pResBlock = createDataBlock(); SSDataBlock* pResBlock = createDataBlock();
if (pResBlock == NULL) { if (pResBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -876,7 +876,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
if (p1->name != NULL) { if (p1->name != NULL) {
STR_TO_VARSTR(str, p1->name); STR_TO_VARSTR(str, p1->name);
} else { // name is not retrieved during filter } else { // name is not retrieved during filter
metaGetTableNameByUid(metaHandle, p1->uid, str); pStorageAPI->metaFn.getTableNameByUid(pVnode, p1->uid, str);
} }
colDataSetVal(pColInfo, i, str, false); colDataSetVal(pColInfo, i, str, false);
@ -889,7 +889,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
if (p1->pTagVal == NULL) { if (p1->pTagVal == NULL) {
colDataSetNULL(pColInfo, i); colDataSetNULL(pColInfo, i);
} else { } else {
const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); const char* p = pStorageAPI->metaFn.extractTagVal(p1->pTagVal, pColInfo->info.type, &tagVal);
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) { if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
colDataSetNULL(pColInfo, i); colDataSetNULL(pColInfo, i);
@ -949,13 +949,13 @@ static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) {
} }
} }
static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle, static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* pVnode,
SIdxFltStatus status) { SIdxFltStatus status, SStorageAPI* pAPI) {
if (pTagCond == NULL) { if (pTagCond == NULL) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
terrno = TDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SArray* pBlockList = NULL; SArray* pBlockList = NULL;
@ -985,7 +985,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
FilterCondType condType = checkTagCond(pTagCond); FilterCondType condType = checkTagCond(pTagCond);
int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond); int32_t filter = optimizeTbnameInCond(pVnode, pListInfo->idInfo.suid, pUidTagList, pTagCond, pAPI);
if (filter == 0) { // tbname in filter is activated, do nothing and return if (filter == 0) { // tbname in filter is activated, do nothing and return
taosArrayClear(pUidList); taosArrayClear(pUidList);
@ -998,9 +998,9 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
terrno = 0; terrno = 0;
} else { } else {
if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) {
code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList); code = pAPI->metaFn.getTableTagsByUid(pVnode, pListInfo->idInfo.suid, pUidTagList);
} else { } else {
code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList); code = pAPI->metaFn.getTableTags(pVnode, pListInfo->idInfo.suid, pUidTagList);
} }
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid); qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid);
@ -1014,7 +1014,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN
goto end; goto end;
} }
pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, pVnode, pAPI);
if (pResBlock == NULL) { if (pResBlock == NULL) {
code = terrno; code = terrno;
goto end; goto end;
@ -1052,8 +1052,8 @@ end:
return code; return code;
} }
int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond,
STableListInfo* pListInfo, uint8_t* digest, const char* idstr) { STableListInfo* pListInfo, uint8_t* digest, const char* idstr, SStorageAPI* pStorageAPI) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
size_t numOfTables = 0; size_t numOfTables = 0;
@ -1065,10 +1065,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
SIdxFltStatus status = SFLT_NOT_INDEX; SIdxFltStatus status = SFLT_NOT_INDEX;
if (pScanNode->tableType != TSDB_SUPER_TABLE) { if (pScanNode->tableType != TSDB_SUPER_TABLE) {
pListInfo->idInfo.uid = pScanNode->uid; pListInfo->idInfo.uid = pScanNode->uid;
if (metaIsTableExist(metaHandle, pScanNode->uid)) { if (pStorageAPI->metaFn.isTableExisted(pVnode, pScanNode->uid)) {
taosArrayPush(pUidList, &pScanNode->uid); taosArrayPush(pUidList, &pScanNode->uid);
} }
code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _end; goto _end;
} }
@ -1080,7 +1080,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
genTagFilterDigest(pTagCond, &context); genTagFilterDigest(pTagCond, &context);
bool acquired = false; bool acquired = false;
metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, pStorageAPI->metaFn.getCachedTableList(pVnode, pScanNode->suid, context.digest, tListLen(context.digest), pUidList,
&acquired); &acquired);
if (acquired) { if (acquired) {
digest[0] = 1; digest[0] = 1;
@ -1091,26 +1091,28 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
} }
if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table
vnodeGetCtbIdList(pVnode, pScanNode->suid, pUidList); // vnodeGetCtbIdList();
pStorageAPI->metaFn.storeGetChildTableList(pVnode, pScanNode->suid, pUidList);
} else { } else {
// failed to find the result in the cache, let try to calculate the results // failed to find the result in the cache, let try to calculate the results
if (pTagIndexCond) { if (pTagIndexCond) {
void* pIndex = tsdbGetIvtIdx(metaHandle); void* pIndex = pStorageAPI->metaFn.storeGetInvertIndex(pVnode);
SIndexMetaArg metaArg = { SIndexMetaArg metaArg = {
.metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; .metaEx = pVnode, .idx = pStorageAPI->metaFn.storeGetIndexInfo(pVnode), .ivtIdx = pIndex, .suid = pScanNode->uid};
status = SFLT_NOT_INDEX; status = SFLT_NOT_INDEX;
code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status);
if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake
qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid); qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid);
code = TDB_CODE_SUCCESS; code = TSDB_CODE_SUCCESS;
} else { } else {
qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList)); qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList));
} }
} }
} }
code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); code = doFilterByTagCond(pListInfo, pUidList, pTagCond, pVnode, status, pStorageAPI);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _end; goto _end;
} }
@ -1127,7 +1129,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
memcpy(pPayload + sizeof(int32_t), taosArrayGet(pUidList, 0), numOfTables * sizeof(uint64_t)); memcpy(pPayload + sizeof(int32_t), taosArrayGet(pUidList, 0), numOfTables * sizeof(uint64_t));
} }
metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); // metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1);
digest[0] = 1; digest[0] = 1;
memcpy(digest + 1, context.digest, tListLen(context.digest)); memcpy(digest + 1, context.digest, tListLen(context.digest));
} }
@ -1164,11 +1166,13 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) {
return keyLen; return keyLen;
} }
int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) { int32_t getGroupIdFromTagsVal(void* pVnode, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId,
SStorageAPI* pAPI) {
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pMeta, 0);
if (metaGetTableEntryByUidCache(&mr, uid) != 0) { // table not exist pAPI->metaReaderFn.initReader(&mr, pVnode, 0);
metaReaderClear(&mr); if (pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, uid) != 0) { // table not exist
pAPI->metaReaderFn.clearReader(&mr);
return TSDB_CODE_PAR_TABLE_NOT_EXIST; return TSDB_CODE_PAR_TABLE_NOT_EXIST;
} }
@ -1187,7 +1191,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode,
REPLACE_NODE(pNew); REPLACE_NODE(pNew);
} else { } else {
nodesDestroyList(groupNew); nodesDestroyList(groupNew);
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
return code; return code;
} }
@ -1204,7 +1208,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode,
if (tTagIsJson(data)) { if (tTagIsJson(data)) {
terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
nodesDestroyList(groupNew); nodesDestroyList(groupNew);
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
return terrno; return terrno;
} }
int32_t len = getJsonValueLen(data); int32_t len = getJsonValueLen(data);
@ -1224,7 +1228,7 @@ int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode,
*pGroupId = calcGroupId(keyBuf, len); *pGroupId = calcGroupId(keyBuf, len);
nodesDestroyList(groupNew); nodesDestroyList(groupNew);
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2030,11 +2034,11 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups); memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups);
taosArrayDestroy(pList); taosArrayDestroy(pList);
return TDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group, int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SScanPhysiNode* pScanNode, SNodeList* group,
bool groupSort, uint8_t *digest) { bool groupSort, uint8_t *digest, SStorageAPI* pAPI) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
bool groupByTbname = groupbyTbname(group); bool groupByTbname = groupbyTbname(group);
@ -2054,7 +2058,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
pTableListInfo->numOfOuputGroups = 1; pTableListInfo->numOfOuputGroups = 1;
} }
} else { } else {
code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest); code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo, digest, pAPI);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -2086,7 +2090,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
} }
uint8_t digest[17] = {0}; uint8_t digest[17] = {0};
int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr); int32_t code = getTableList(pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo, digest, idStr, &pTaskInfo->storageAPI);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to getTableList, code: %s", tstrerror(code)); qError("failed to getTableList, code: %s", tstrerror(code));
return code; return code;
@ -2104,7 +2108,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest); code = buildGroupIdMapForAllTables(pTableListInfo, pHandle, pScanNode, pGroupTags, groupSort, digest, &pTaskInfo->storageAPI);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }

67
source/libs/executor/src/executor.c
View File

@ -14,6 +14,8 @@
*/ */
#include "executor.h" #include "executor.h"
#include <libs/transport/trpc.h>
#include <libs/wal/wal.h>
#include "executorInt.h" #include "executorInt.h"
#include "operator.h" #include "operator.h"
#include "planner.h" #include "planner.h"
@ -21,7 +23,8 @@
#include "tdatablock.h" #include "tdatablock.h"
#include "tref.h" #include "tref.h"
#include "tudf.h" #include "tudf.h"
#include "vnode.h"
#include "storageapi.h"
static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT;
int32_t exchangeObjRefPool = -1; int32_t exchangeObjRefPool = -1;
@ -156,18 +159,18 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
} }
} }
void doSetTaskId(SOperatorInfo* pOperator) { void doSetTaskId(SOperatorInfo* pOperator, SStorageAPI *pAPI) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamScanInfo* pStreamScanInfo = pOperator->info; SStreamScanInfo* pStreamScanInfo = pOperator->info;
if (pStreamScanInfo->pTableScanOp != NULL) { if (pStreamScanInfo->pTableScanOp != NULL) {
STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info;
if (pScanInfo->base.dataReader != NULL) { if (pScanInfo->base.dataReader != NULL) {
tsdbReaderSetId(pScanInfo->base.dataReader, pTaskInfo->id.str); pAPI->storeReader.setReaderId(pScanInfo->base.dataReader, pTaskInfo->id.str);
} }
} }
} else { } else {
doSetTaskId(pOperator->pDownstream[0]); doSetTaskId(pOperator->pDownstream[0], pAPI);
} }
} }
@ -177,7 +180,7 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) {
buildTaskId(taskId, queryId, pTaskInfo->id.str); buildTaskId(taskId, queryId, pTaskInfo->id.str);
// set the idstr for tsdbReader // set the idstr for tsdbReader
doSetTaskId(pTaskInfo->pRoot); doSetTaskId(pTaskInfo->pRoot, &pTaskInfo->storageAPI);
} }
int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) {
@ -320,7 +323,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v
return pTaskInfo; return pTaskInfo;
} }
static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr,
SStorageAPI* pAPI) {
SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
int32_t numOfUids = taosArrayGetSize(tableIdList); int32_t numOfUids = taosArrayGetSize(tableIdList);
if (numOfUids == 0) { if (numOfUids == 0) {
@ -336,11 +340,11 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
// let's discard the tables those are not created according to the queried super table. // let's discard the tables those are not created according to the queried super table.
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.meta, 0);
for (int32_t i = 0; i < numOfUids; ++i) { for (int32_t i = 0; i < numOfUids; ++i) {
uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i);
int32_t code = metaGetTableEntryByUid(&mr, *id); int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, *id);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr); qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr);
continue; continue;
@ -368,7 +372,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
if (pScanInfo->pTagCond != NULL) { if (pScanInfo->pTagCond != NULL) {
bool qualified = false; bool qualified = false;
STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid}; STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid};
code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified, pAPI);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr);
continue; continue;
@ -383,7 +387,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
taosArrayPush(qa, id); taosArrayPush(qa, id);
} }
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
return qa; return qa;
} }
@ -401,10 +405,10 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
SStreamScanInfo* pScanInfo = pInfo->info; SStreamScanInfo* pScanInfo = pInfo->info;
if (isAdd) { // add new table id if (isAdd) { // add new table id
SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id); SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id, &pTaskInfo->storageAPI);
int32_t numOfQualifiedTables = taosArrayGetSize(qa); int32_t numOfQualifiedTables = taosArrayGetSize(qa);
qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id); qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id);
code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); code = pTaskInfo->storageAPI.tqReaderFn.tqReaderAddTables(pScanInfo->tqReader, qa);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(qa); taosArrayDestroy(qa);
return code; return code;
@ -434,7 +438,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
keyInfo.groupId = keyInfo.uid; keyInfo.groupId = keyInfo.uid;
} else { } else {
code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf,
&keyInfo.groupId); &keyInfo.groupId, &pTaskInfo->storageAPI);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(keyBuf); taosMemoryFree(keyBuf);
taosArrayDestroy(qa); taosArrayDestroy(qa);
@ -456,7 +460,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI
} else { // remove the table id in current list } else { // remove the table id in current list
qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id); qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id);
taosWLockLatch(&pTaskInfo->lock); taosWLockLatch(&pTaskInfo->lock);
code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); code = pTaskInfo->storageAPI.tqReaderFn.tqReaderRemoveTables(pScanInfo->tqReader, tableIdList);
taosWUnLockLatch(&pTaskInfo->lock); taosWUnLockLatch(&pTaskInfo->lock);
} }
@ -1060,6 +1064,8 @@ void qStreamSetOpen(qTaskInfo_t tinfo) {
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SOperatorInfo* pOperator = pTaskInfo->pRoot; SOperatorInfo* pOperator = pTaskInfo->pRoot;
const char* id = GET_TASKID(pTaskInfo); const char* id = GET_TASKID(pTaskInfo);
@ -1081,14 +1087,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
if (pOffset->type == TMQ_OFFSET__LOG) { if (pOffset->type == TMQ_OFFSET__LOG) {
// todo refactor: move away // todo refactor: move away
tsdbReaderClose(pScanBaseInfo->dataReader); pTaskInfo->storageAPI.storeReader.storeReaderClose(pScanBaseInfo->dataReader);
pScanBaseInfo->dataReader = NULL; pScanBaseInfo->dataReader = NULL;
walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset); ASSERT(0);
if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { // walReaderVerifyOffset(pInfo->tqReader->pWalReader, pOffset);
qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); // if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) {
return -1; // qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
} // return -1;
// }
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
// iterate all tables from tableInfoList, and retrieve rows from each table one-by-one // iterate all tables from tableInfoList, and retrieve rows from each table one-by-one
// those data are from the snapshot in tsdb, besides the data in the wal file. // those data are from the snapshot in tsdb, besides the data in the wal file.
@ -1141,8 +1148,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
pScanInfo->scanTimes = 0; pScanInfo->scanTimes = 0;
if (pScanBaseInfo->dataReader == NULL) { if (pScanBaseInfo->dataReader == NULL) {
int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, int32_t code = pTaskInfo->storageAPI.storeReader.storeReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1,
pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false, NULL); pScanInfo->pResBlock, (void**) &pScanBaseInfo->dataReader, id, false, NULL);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id);
terrno = code; terrno = code;
@ -1152,8 +1159,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s",
uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
} else { } else {
tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); pTaskInfo->storageAPI.storeReader.storeReaderSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1);
tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); pTaskInfo->storageAPI.storeReader.storeReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond);
qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s",
uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id);
} }
@ -1175,14 +1182,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id);
STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo; STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo;
if (setForSnapShot(sContext, pOffset->uid) != 0) { if (pAPI->snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) {
qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id); qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR; terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1; return -1;
} }
SMetaTableInfo mtInfo = getUidfromSnapShot(sContext); SMetaTableInfo mtInfo = pTaskInfo->storageAPI.snapshotFn.storeSSGetTableInfo(sContext);
tsdbReaderClose(pInfo->dataReader); pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader);
pInfo->dataReader = NULL; pInfo->dataReader = NULL;
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
@ -1200,7 +1207,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0);
int32_t size = tableListGetSize(pTableListInfo); int32_t size = tableListGetSize(pTableListInfo);
tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL, pTaskInfo->storageAPI.storeReader.storeReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, (void**) &pInfo->dataReader, NULL,
false, NULL); false, NULL);
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
@ -1212,7 +1219,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
} else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
SStreamRawScanInfo* pInfo = pOperator->info; SStreamRawScanInfo* pInfo = pOperator->info;
SSnapContext* sContext = pInfo->sContext; SSnapContext* sContext = pInfo->sContext;
if (setForSnapShot(sContext, pOffset->uid) != 0) { if (pTaskInfo->storageAPI.snapshotFn.storeCreateSnapshot(sContext, pOffset->uid) != 0) {
qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version);
terrno = TSDB_CODE_PAR_INTERNAL_ERROR; terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
return -1; return -1;
@ -1221,7 +1228,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
id); id);
} else if (pOffset->type == TMQ_OFFSET__LOG) { } else if (pOffset->type == TMQ_OFFSET__LOG) {
SStreamRawScanInfo* pInfo = pOperator->info; SStreamRawScanInfo* pInfo = pOperator->info;
tsdbReaderClose(pInfo->dataReader); pTaskInfo->storageAPI.storeReader.storeReaderClose(pInfo->dataReader);
pInfo->dataReader = NULL; pInfo->dataReader = NULL;
qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id);
} }

141
source/libs/executor/src/executorInt.c
View File

@ -33,7 +33,7 @@
#include "tcompare.h" #include "tcompare.h"
#include "thash.h" #include "thash.h"
#include "ttypes.h" #include "ttypes.h"
#include "vnode.h" #include "storageapi.h"
#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN)
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
@ -844,6 +844,8 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
SDiskbasedBuf* pBuf) { SDiskbasedBuf* pBuf) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSDataBlock* pBlock = pbInfo->pRes; SSDataBlock* pBlock = pbInfo->pRes;
// set output datablock version // set output datablock version
@ -860,12 +862,13 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr
doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false); doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold, false);
void* tbname = NULL; void* tbname = NULL;
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else { } else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} }
streamFreeVal(tbname);
pAPI->stateStore.streamStateFreeVal(tbname);
} }
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
@ -1066,122 +1069,22 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t resultRowEncode(void* k, int32_t* size, char* buf) { int32_t releaseOutputBuf(void* pState, SWinKey* pKey, SResultRow* pResult, SStateStore* pAPI) {
// SResultRow* key = k; pAPI->streamStateReleaseBuf(pState, pKey, pResult);
// int len = 0;
// int struLen = *size;
// len += taosEncodeFixedI32((void**)&buf, key->pageId);
// uint32_t offset = key->offset;
// len += taosEncodeFixedU32((void**)&buf, offset);
// len += taosEncodeFixedI8((void**)&buf, key->startInterp);
// len += taosEncodeFixedI8((void**)&buf, key->endInterp);
// len += taosEncodeFixedI8((void**)&buf, key->closed);
// len += taosEncodeFixedU32((void**)&buf, key->numOfRows);
// len += taosEncodeFixedI64((void**)&buf, key->win.skey);
// len += taosEncodeFixedI64((void**)&buf, key->win.ekey);
// int32_t numOfEntryInfo = (struLen - sizeof(SResultRow)) / sizeof(struct SResultRowEntryInfo);
// len += taosEncodeFixedI32((void**)&buf, numOfEntryInfo);
// for (int i = 0; i < numOfEntryInfo; i++) {
// SResultRowEntryInfo* p = &key->pEntryInfo[i];
// uint8_t value = p->initialized ? 1 : 0;
// len += taosEncodeFixedU8((void**)&buf, value);
// value = p->complete ? 1 : 0;
// len += taosEncodeFixedU8((void**)&buf, value);
// value = p->isNullRes;
// len += taosEncodeFixedU8((void**)&buf, value);
// len += taosEncodeFixedU16((void**)&buf, p->numOfRes);
// }
// {
// char* strBuf = taosMemoryCalloc(1, *size * 100);
// resultRowToString(key, *size, strBuf);
// qWarn("encode result row:%s", strBuf);
// }
// return len;
return 0;
}
int32_t resultRowDecode(void** k, size_t size, char* buf) {
// char* p1 = buf;
// int32_t numOfEntryInfo = 0;
// uint32_t entryOffset = sizeof(int32_t) + sizeof(uint32_t) + sizeof(int8_t) + sizeof(int8_t) + sizeof(int8_t) +
// sizeof(uint32_t) + sizeof(int64_t) + sizeof(int64_t);
// taosDecodeFixedI32(p1 + entryOffset, &numOfEntryInfo);
// char* p = buf;
// size = sizeof(SResultRow) + numOfEntryInfo * sizeof(SResultRowEntryInfo);
// SResultRow* key = taosMemoryCalloc(1, size);
// p = taosDecodeFixedI32(p, (int32_t*)&key->pageId);
// uint32_t offset = 0;
// p = taosDecodeFixedU32(p, &offset);
// key->offset = offset;
// p = taosDecodeFixedI8(p, (int8_t*)(&key->startInterp));
// p = taosDecodeFixedI8(p, (int8_t*)(&key->endInterp));
// p = taosDecodeFixedI8(p, (int8_t*)&key->closed);
// p = taosDecodeFixedU32(p, &key->numOfRows);
// p = taosDecodeFixedI64(p, &key->win.skey);
// p = taosDecodeFixedI64(p, &key->win.ekey);
// p = taosDecodeFixedI32(p, &numOfEntryInfo);
// for (int i = 0; i < numOfEntryInfo; i++) {
// SResultRowEntryInfo* pInfo = &key->pEntryInfo[i];
// uint8_t value = 0;
// p = taosDecodeFixedU8(p, &value);
// pInfo->initialized = (value == 1) ? true : false;
// p = taosDecodeFixedU8(p, &value);
// pInfo->complete = (value == 1) ? true : false;
// p = taosDecodeFixedU8(p, &value);
// pInfo->isNullRes = value;
// p = taosDecodeFixedU16(p, &pInfo->numOfRes);
// }
// *k = key;
// {
// char* strBuf = taosMemoryCalloc(1, size * 100);
// resultRowToString(key, size, strBuf);
// qWarn("decode result row:%s", strBuf);
// }
// return size;
return 0;
}
int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize) {
// char* buf = taosMemoryCalloc(1, resSize * 10);
// int len = resultRowEncode((void*)pResult, &resSize, buf);
// char* buf = taosMemoryCalloc(1, resSize);
// memcpy(buf, pResult, resSize);
streamStatePut(pState, pKey, (char*)pResult, resSize);
// taosMemoryFree(buf);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult) { int32_t saveSessionDiscBuf(void* pState, SSessionKey* key, void* buf, int32_t size, SStateStore* pAPI) {
streamStateReleaseBuf(pState, pKey, pResult); pAPI->streamStateSessionPut(pState, key, (const void*)buf, size);
releaseOutputBuf(pState, NULL, (SResultRow*)buf, pAPI);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size) { int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
streamStateSessionPut(pState, key, (const void*)buf, size);
releaseOutputBuf(pState, NULL, (SResultRow*)buf);
return TSDB_CODE_SUCCESS;
}
int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock,
SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { SExprSupp* pSup, SGroupResInfo* pGroupResInfo) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SExprInfo* pExprInfo = pSup->pExprInfo; SExprInfo* pExprInfo = pSup->pExprInfo;
int32_t numOfExprs = pSup->numOfExprs; int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
@ -1193,7 +1096,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
SSessionKey* pKey = taosArrayGet(pGroupResInfo->pRows, i); SSessionKey* pKey = taosArrayGet(pGroupResInfo->pRows, i);
int32_t size = 0; int32_t size = 0;
void* pVal = NULL; void* pVal = NULL;
int32_t code = streamStateSessionGet(pState, pKey, &pVal, &size); int32_t code = pAPI->stateStore.streamStateSessionGet(pState, pKey, &pVal, &size);
ASSERT(code == 0); ASSERT(code == 0);
if (code == -1) { if (code == -1) {
// coverity scan // coverity scan
@ -1205,7 +1108,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
// no results, continue to check the next one // no results, continue to check the next one
if (pRow->numOfRows == 0) { if (pRow->numOfRows == 0) {
pGroupResInfo->index += 1; pGroupResInfo->index += 1;
releaseOutputBuf(pState, NULL, pRow); releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
continue; continue;
} }
@ -1213,23 +1116,23 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
pBlock->info.id.groupId = pKey->groupId; pBlock->info.id.groupId = pKey->groupId;
void* tbname = NULL; void* tbname = NULL;
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else { } else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} }
streamFreeVal(tbname); pAPI->stateStore.streamStateFreeVal(tbname);
} else { } else {
// current value belongs to different group, it can't be packed into one datablock // current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.id.groupId != pKey->groupId) { if (pBlock->info.id.groupId != pKey->groupId) {
releaseOutputBuf(pState, NULL, pRow); releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
break; break;
} }
} }
if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
ASSERT(pBlock->info.rows > 0); ASSERT(pBlock->info.rows > 0);
releaseOutputBuf(pState, NULL, pRow); releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
break; break;
} }
@ -1260,7 +1163,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
pBlock->info.dataLoad = 1; pBlock->info.dataLoad = 1;
pBlock->info.rows += pRow->numOfRows; pBlock->info.rows += pRow->numOfRows;
releaseOutputBuf(pState, NULL, pRow); releaseOutputBuf(pState, NULL, pRow, &pAPI->stateStore);
} }
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1292,7 +1195,7 @@ void qStreamCloseTsdbReader(void* task) {
qDebug("wait for the reader stopping"); qDebug("wait for the reader stopping");
} }
tsdbReaderClose(pTSInfo->base.dataReader); pTaskInfo->storageAPI.storeReader.storeReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL; pTSInfo->base.dataReader = NULL;
// restore the status, todo refactor. // restore the status, todo refactor.

85
source/libs/executor/src/filloperator.c
View File

@ -443,7 +443,8 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols);
tSimpleHashCleanup(pFillSup->pResMap); tSimpleHashCleanup(pFillSup->pResMap);
pFillSup->pResMap = NULL; pFillSup->pResMap = NULL;
releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); ASSERT(0);
releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pFillSup->pAPI->stateStore); //?????
pFillSup->cur.pRowVal = NULL; pFillSup->cur.pRowVal = NULL;
cleanupExprSupp(&pFillSup->notFillExprSup); cleanupExprSupp(&pFillSup->notFillExprSup);
@ -490,74 +491,79 @@ static void resetFillWindow(SResultRowData* pRowData) {
pRowData->pRowVal = NULL; pRowData->pRowVal = NULL;
} }
void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) {
resetFillWindow(&pFillSup->prev); resetFillWindow(&pFillSup->prev);
releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); ASSERT(0);
releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal, &pAPI->stateStore); //???
resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->cur);
resetFillWindow(&pFillSup->next); resetFillWindow(&pFillSup->next);
resetFillWindow(&pFillSup->nextNext); resetFillWindow(&pFillSup->nextNext);
} }
void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
resetPrevAndNextWindow(pFillSup, pState);
void* pState = pOperator->pTaskInfo->streamInfo.pState;
resetPrevAndNextWindow(pFillSup, pState, pAPI);
SWinKey key = {.ts = ts, .groupId = groupId}; SWinKey key = {.ts = ts, .groupId = groupId};
int32_t curVLen = 0; int32_t curVLen = 0;
int32_t code = streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen);
int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen);
ASSERT(code == TSDB_CODE_SUCCESS); ASSERT(code == TSDB_CODE_SUCCESS);
pFillSup->cur.key = key.ts; pFillSup->cur.key = key.ts;
} }
void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
resetPrevAndNextWindow(pFillSup, pState); void* pState = pOperator->pTaskInfo->streamInfo.pState;
resetPrevAndNextWindow(pFillSup, pState, pAPI);
SWinKey key = {.ts = ts, .groupId = groupId}; SWinKey key = {.ts = ts, .groupId = groupId};
void* curVal = NULL; void* curVal = NULL;
int32_t curVLen = 0; int32_t curVLen = 0;
int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen);
ASSERT(code == TSDB_CODE_SUCCESS); ASSERT(code == TSDB_CODE_SUCCESS);
pFillSup->cur.key = key.ts; pFillSup->cur.key = key.ts;
pFillSup->cur.pRowVal = curVal; pFillSup->cur.pRowVal = curVal;
SStreamStateCur* pCur = streamStateFillSeekKeyPrev(pState, &key); void* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key);
SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId}; SWinKey preKey = {.ts = INT64_MIN, .groupId = groupId};
void* preVal = NULL; void* preVal = NULL;
int32_t preVLen = 0; int32_t preVLen = 0;
code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pFillSup->prev.key = preKey.ts; pFillSup->prev.key = preKey.ts;
pFillSup->prev.pRowVal = preVal; pFillSup->prev.pRowVal = preVal;
code = streamStateCurNext(pState, pCur); code = pAPI->stateStore.streamStateCurNext(pState, pCur);
ASSERT(code == TSDB_CODE_SUCCESS); ASSERT(code == TSDB_CODE_SUCCESS);
code = streamStateCurNext(pState, pCur); code = pAPI->stateStore.streamStateCurNext(pState, pCur);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
pCur = NULL; pCur = NULL;
} }
} else { } else {
streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
pCur = streamStateFillSeekKeyNext(pState, &key); pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key);
} }
SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId}; SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId};
void* nextVal = NULL; void* nextVal = NULL;
int32_t nextVLen = 0; int32_t nextVLen = 0;
code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pFillSup->next.key = nextKey.ts; pFillSup->next.key = nextKey.ts;
pFillSup->next.pRowVal = nextVal; pFillSup->next.pRowVal = nextVal;
if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) {
code = streamStateCurNext(pState, pCur); code = pAPI->stateStore.streamStateCurNext(pState, pCur);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
SWinKey nextNextKey = {.groupId = groupId}; SWinKey nextNextKey = {.groupId = groupId};
void* nextNextVal = NULL; void* nextNextVal = NULL;
int32_t nextNextVLen = 0; int32_t nextNextVLen = 0;
code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pFillSup->nextNext.key = nextNextKey.ts; pFillSup->nextNext.key = nextNextKey.ts;
pFillSup->nextNext.pRowVal = nextNextVal; pFillSup->nextNext.pRowVal = nextNextVal;
@ -565,7 +571,7 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId,
} }
} }
} }
streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
} }
static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; }
@ -922,8 +928,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
} }
static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) {
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SWinKey key = {.groupId = groupId, .ts = pRow->key}; SWinKey key = {.groupId = groupId, .ts = pRow->key};
int32_t code = streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len);
qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code); qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 " code:%d", key.ts, key.groupId, code);
ASSERT(code == TSDB_CODE_SUCCESS); ASSERT(code == TSDB_CODE_SUCCESS);
} }
@ -1021,7 +1029,8 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) {
} }
static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) {
SStreamState* pState = pOp->pTaskInfo->streamInfo.pState; SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;
void* pState = pOp->pTaskInfo->streamInfo.pState;
SSDataBlock* pBlock = delRes; SSDataBlock* pBlock = delRes;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
@ -1041,14 +1050,14 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_
SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
void* tbname = NULL; void* tbname = NULL;
streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname);
if (tbname == NULL) { if (tbname == NULL) {
colDataSetNULL(pTableCol, pBlock->info.rows); colDataSetNULL(pTableCol, pBlock->info.rows);
} else { } else {
char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
streamFreeVal(tbname); pAPI->stateStore.streamStateFreeVal(tbname);
} }
pBlock->info.rows++; pBlock->info.rows++;
@ -1070,12 +1079,13 @@ static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY end
} }
static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) {
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillOperatorInfo* pInfo = pOperator->info;
getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup);
setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo);
SWinKey key = {.ts = startTs, .groupId = groupId}; SWinKey key = {.ts = startTs, .groupId = groupId};
if (!pInfo->pFillInfo->needFill) { if (!pInfo->pFillInfo->needFill) {
streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
} else { } else {
STimeRange tw = { STimeRange tw = {
@ -1093,6 +1103,8 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE
} }
static void doDeleteFillFinalize(SOperatorInfo* pOperator) { static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillOperatorInfo* pInfo = pOperator->info;
SStreamFillInfo* pFillInfo = pInfo->pFillInfo; SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
int32_t size = taosArrayGetSize(pFillInfo->delRanges); int32_t size = taosArrayGetSize(pFillInfo->delRanges);
@ -1109,17 +1121,16 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
pInfo->pRes->info.id.groupId = range->groupId; pInfo->pRes->info.id.groupId = range->groupId;
} }
SWinKey key = {.ts = range->skey, .groupId = range->groupId}; SWinKey key = {.ts = range->skey, .groupId = range->groupId};
streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
} }
} }
static void doDeleteFillResult(SOperatorInfo* pOperator) { static void doDeleteFillResult(SOperatorInfo* pOperator) {
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillOperatorInfo* pInfo = pOperator->info;
SStreamFillSupporter* pFillSup = pInfo->pFillSup;
SStreamFillInfo* pFillInfo = pInfo->pFillInfo; SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
SSDataBlock* pBlock = pInfo->pSrcDelBlock; SSDataBlock* pBlock = pInfo->pSrcDelBlock;
SSDataBlock* pRes = pInfo->pRes;
SSDataBlock* pDelRes = pInfo->pDelRes;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
TSKEY* tsStarts = (TSKEY*)pStartCol->pData; TSKEY* tsStarts = (TSKEY*)pStartCol->pData;
@ -1130,7 +1141,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
TSKEY endTs = ts; TSKEY endTs = ts;
uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; uint64_t groupId = groupIds[pInfo->srcDelRowIndex];
SWinKey key = {.ts = ts, .groupId = groupId}; SWinKey key = {.ts = ts, .groupId = groupId};
SStreamStateCur* pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); void* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key);
if (!pCur) { if (!pCur) {
pInfo->srcDelRowIndex++; pInfo->srcDelRowIndex++;
continue; continue;
@ -1152,15 +1163,15 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; SWinKey delKey = {.groupId = delGroupId, .ts = delTs};
if (delTs == nextKey.ts) { if (delTs == nextKey.ts) {
code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen);
} }
// ts will be deleted later // ts will be deleted later
if (delTs != ts) { if (delTs != ts) {
streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey);
streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
} }
endTs = TMAX(delTs, nextKey.ts - 1); endTs = TMAX(delTs, nextKey.ts - 1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -1169,9 +1180,11 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) {
} }
pInfo->srcDelRowIndex++; pInfo->srcDelRowIndex++;
} }
streamStateFreeCur(pCur);
pAPI->stateStore.streamStateFreeCur(pCur);
doDeleteFillResultImpl(pOperator, ts, endTs, groupId); doDeleteFillResultImpl(pOperator, ts, endTs, groupId);
} }
pFillInfo->current = pFillInfo->end + 1; pFillInfo->current = pFillInfo->end + 1;
} }

27
source/libs/executor/src/groupoperator.c
View File

@ -950,6 +950,8 @@ static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo
static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; }
static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamPartitionOperatorInfo* pInfo = pOperator->info; SStreamPartitionOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pDest = pInfo->binfo.pRes; SSDataBlock* pDest = pInfo->binfo.pRes;
ASSERT(hasRemainPartion(pInfo)); ASSERT(hasRemainPartion(pInfo));
@ -972,9 +974,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
pDest->info.parTbName[0] = 0; pDest->info.parTbName[0] = 0;
if (pInfo->tbnameCalSup.numOfExprs > 0) { if (pInfo->tbnameCalSup.numOfExprs > 0) {
void* tbname = NULL; void* tbname = NULL;
if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { if (pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) {
memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
streamFreeVal(tbname); pAPI->stateStore.streamStateFreeVal(tbname);
} }
} }
taosArrayDestroy(pParInfo->rowIds); taosArrayDestroy(pParInfo->rowIds);
@ -990,10 +992,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
return pDest; return pDest;
} }
void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId,
SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) { SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) {
void* pValue = NULL; void* pValue = NULL;
if (streamStateGetParName(pState, groupId, &pValue) != 0) { if (pAPI->streamStateGetParName(pState, groupId, &pValue) != 0) {
SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId);
memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN);
pTmpBlock->info.id.groupId = groupId; pTmpBlock->info.id.groupId = groupId;
@ -1010,7 +1012,7 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1);
len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1);
memcpy(tbName, varDataVal(pData), len); memcpy(tbName, varDataVal(pData), len);
streamStatePutParName(pState, groupId, tbName); pAPI->streamStatePutParName(pState, groupId, tbName);
} }
memcpy(pTmpBlock->info.parTbName, tbName, len); memcpy(pTmpBlock->info.parTbName, tbName, len);
pDestBlock->info.rows--; pDestBlock->info.rows--;
@ -1034,10 +1036,12 @@ void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp*
} else { } else {
memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN); memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN);
} }
streamStateReleaseBuf(pState, NULL, pValue); pAPI->streamStateReleaseBuf(pState, NULL, pValue);
} }
static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) {
SExecTaskInfo* pTask = pOperator->pTaskInfo;
SStreamPartitionOperatorInfo* pInfo = pOperator->info; SStreamPartitionOperatorInfo* pInfo = pOperator->info;
if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) ||
taosHashGetSize(pInfo->pPartitions) == 0) { taosHashGetSize(pInfo->pPartitions) == 0) {
@ -1050,8 +1054,8 @@ static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) {
if (pInfo->pTbNameIte != NULL) { if (pInfo->pTbNameIte != NULL) {
SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte;
int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0); int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0);
appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup,
pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes); pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore);
pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte);
} }
return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL; return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL;
@ -1164,14 +1168,17 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
} }
void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) { void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) {
SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI;
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
return; return;
} }
SStreamScanInfo* pScanInfo = downstream->info; SStreamScanInfo* pScanInfo = downstream->info;
pScanInfo->partitionSup = *pParSup; pScanInfo->partitionSup = *pParSup;
pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartScalarSup = pExpr;
if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0); pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0);
} }
} }

14
source/libs/executor/src/operator.c
View File

@ -25,7 +25,8 @@
#include "operator.h" #include "operator.h"
#include "query.h" #include "query.h"
#include "querytask.h" #include "querytask.h"
#include "vnode.h"
#include "storageapi.h"
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn,
@ -233,11 +234,12 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
} }
static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) { static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) {
SStorageAPI* pAPI = pParam->pParam;
if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pInfo = pOperator->info; STableScanInfo* pInfo = pOperator->info;
if (pInfo->base.dataReader != NULL) { if (pInfo->base.dataReader != NULL) {
tsdbReaderSetCloseFlag(pInfo->base.dataReader); pAPI->storeReader.storeReaderNotifyClosing(pInfo->base.dataReader);
} }
return OPTR_FN_RET_ABORT; return OPTR_FN_RET_ABORT;
} else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
@ -246,7 +248,7 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam,
if (pInfo->pTableScanOp != NULL) { if (pInfo->pTableScanOp != NULL) {
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) { if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) {
tsdbReaderSetCloseFlag(pTableScanInfo->base.dataReader); pAPI->storeReader.storeReaderNotifyClosing(pTableScanInfo->base.dataReader);
} }
} }
@ -256,8 +258,8 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam,
return OPTR_FN_RET_CONTINUE; return OPTR_FN_RET_CONTINUE;
} }
int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr) { int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr, SStorageAPI* pAPI) {
STraverParam p = {0}; STraverParam p = {.pParam = pAPI};
traverseOperatorTree(pOperator, doStopDataReader, &p, pIdStr); traverseOperatorTree(pOperator, doStopDataReader, &p, pIdStr);
return p.code; return p.code;
} }
@ -379,7 +381,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
if (pBlockNode->tableType == TSDB_SUPER_TABLE) { if (pBlockNode->tableType == TSDB_SUPER_TABLE) {
SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo));
int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pList); int32_t code = pTaskInfo->storageAPI.metaFn.storeGetTableList(pHandle->vnode, pBlockNode->uid, pList);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = terrno; pTaskInfo->code = terrno;
return NULL; return NULL;

1
source/libs/executor/src/projectoperator.c
View File

@ -18,6 +18,7 @@
#include "functionMgt.h" #include "functionMgt.h"
#include "operator.h" #include "operator.h"
#include "querytask.h" #include "querytask.h"
#include "tdatablock.h"
typedef struct SProjectOperatorInfo { typedef struct SProjectOperatorInfo {
SOptrBasicInfo binfo; SOptrBasicInfo binfo;

18
source/libs/executor/src/querytask.c
View File

@ -29,9 +29,9 @@
#include "operator.h" #include "operator.h"
#include "query.h" #include "query.h"
#include "querytask.h" #include "querytask.h"
#include "storageapi.h"
#include "thash.h" #include "thash.h"
#include "ttypes.h" #include "ttypes.h"
#include "vnode.h"
#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st)))
@ -63,7 +63,7 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); }
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) {
pTaskInfo->code = rspCode; pTaskInfo->code = rspCode;
stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str); stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str, &pTaskInfo->storageAPI);
} }
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
@ -120,13 +120,15 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
return terrno; return terrno;
} }
metaReaderInit(&mr, pHandle->meta, 0); SStorageAPI* pAPI = &pTaskInfo->storageAPI;
int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid);
pAPI->metaReaderFn.initReader(&mr, pHandle->meta, 0);
int32_t code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, pScanNode->uid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid,
GET_TASKID(pTaskInfo)); GET_TASKID(pTaskInfo));
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
return terrno; return terrno;
} }
@ -142,9 +144,9 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
tDecoderClear(&mr.coder); tDecoderClear(&mr.coder);
tb_uid_t suid = mr.me.ctbEntry.suid; tb_uid_t suid = mr.me.ctbEntry.suid;
code = metaGetTableEntryByUidCache(&mr, suid); code = pAPI->metaReaderFn.readerGetEntryGetUidCache(&mr, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
return terrno; return terrno;
} }
@ -154,7 +156,7 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo
pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow); pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow);
} }
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode); pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

354
source/libs/executor/src/scanoperator.c
View File

@ -33,6 +33,9 @@
#include "operator.h" #include "operator.h"
#include "querytask.h" #include "querytask.h"
#include "storageapi.h"
#include "wal.h"
int32_t scanDebug = 0; int32_t scanDebug = 0;
#define MULTI_READER_MAX_TABLE_NUM 5000 #define MULTI_READER_MAX_TABLE_NUM 5000
@ -261,8 +264,10 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA
} }
static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
bool allColumnsHaveAgg = true; bool allColumnsHaveAgg = true;
int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); int32_t code = pAPI->storeReader.storeReaderRetrieveBlockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
@ -323,6 +328,8 @@ bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock, static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
uint32_t* status) { uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
pCost->totalBlocks += 1; pCost->totalBlocks += 1;
@ -343,7 +350,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->filterOutBlocks += 1; pCost->filterOutBlocks += 1;
pCost->totalRows += pBlock->info.rows; pCost->totalRows += pBlock->info.rows;
tsdbReleaseDataBlock(pTableScanInfo->dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64,
@ -351,7 +358,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pBlockInfo->id.uid); pBlockInfo->id.uid);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
pCost->skipBlocks += 1; pCost->skipBlocks += 1;
tsdbReleaseDataBlock(pTableScanInfo->dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) {
pCost->loadBlockStatis += 1; pCost->loadBlockStatis += 1;
@ -361,7 +368,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
tsdbReleaseDataBlock(pTableScanInfo->dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { } else {
qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo));
@ -383,7 +390,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost->filterOutBlocks += 1; pCost->filterOutBlocks += 1;
(*status) = FUNC_DATA_REQUIRED_FILTEROUT; (*status) = FUNC_DATA_REQUIRED_FILTEROUT;
tsdbReleaseDataBlock(pTableScanInfo->dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} }
@ -398,10 +405,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64,
GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pCost->skipBlocks += 1; pCost->skipBlocks += 1;
tsdbReleaseDataBlock(pTableScanInfo->dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->dataReader);
STableScanInfo* pTableScanInfo = pOperator->info; STableScanInfo* p1 = pOperator->info;
if (taosHashGetSize(pTableScanInfo->pIgnoreTables) == taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList)) { if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) {
*status = FUNC_DATA_REQUIRED_ALL_FILTEROUT; *status = FUNC_DATA_REQUIRED_ALL_FILTEROUT;
} else { } else {
*status = FUNC_DATA_REQUIRED_FILTEROUT; *status = FUNC_DATA_REQUIRED_FILTEROUT;
@ -412,7 +419,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost->totalCheckedRows += pBlock->info.rows; pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1; pCost->loadBlocks += 1;
SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); SSDataBlock* p = pAPI->storeReader.storeReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
if (p == NULL) { if (p == NULL) {
return terrno; return terrno;
} }
@ -524,8 +531,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
// 1. check if it is existed in meta cache // 1. check if it is existed in meta cache
if (pCache == NULL) { if (pCache == NULL) {
metaReaderInit(&mr, pHandle->meta, 0); pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0);
code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
// when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed.
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
@ -538,11 +545,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno),
idStr); idStr);
} }
metaReaderClear(&mr); pHandle->api.metaReaderFn.clearReader(&mr);
return terrno; return terrno;
} }
metaReaderReleaseLock(&mr); pHandle->api.metaReaderFn.readerReleaseLock(&mr);
val.pName = mr.me.name; val.pName = mr.me.name;
val.pTags = (STag*)mr.me.ctbEntry.pTags; val.pTags = (STag*)mr.me.ctbEntry.pTags;
@ -553,8 +560,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid));
if (h == NULL) { if (h == NULL) {
metaReaderInit(&mr, pHandle->meta, 0); pHandle->api.metaReaderFn.initReader(&mr, pHandle->meta, 0);
code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); code = pHandle->api.metaReaderFn.readerGetEntryGetUidCache(&mr, pBlock->info.id.uid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",
@ -565,11 +572,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno),
idStr); idStr);
} }
metaReaderClear(&mr); pHandle->api.metaReaderFn.clearReader(&mr);
return terrno; return terrno;
} }
metaReaderReleaseLock(&mr); pHandle->api.metaReaderFn.readerReleaseLock(&mr);
STableCachedVal* pVal = createTableCacheVal(&mr); STableCachedVal* pVal = createTableCacheVal(&mr);
@ -609,7 +616,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
} else { // these are tags } else { // these are tags
STagVal tagVal = {0}; STagVal tagVal = {0};
tagVal.cid = pExpr1->base.pParam[0].pCol->colId; tagVal.cid = pExpr1->base.pParam[0].pCol->colId;
const char* p = metaGetTableTagVal(val.pTags, pColInfoData->info.type, &tagVal); const char* p = pHandle->api.metaFn.extractTagVal(val.pTags, pColInfoData->info.type, &tagVal);
char* data = NULL; char* data = NULL;
if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) {
@ -628,7 +635,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
} }
if (code) { if (code) {
if (freeReader) { if (freeReader) {
metaReaderClear(&mr); pHandle->api.metaReaderFn.clearReader(&mr);
} }
return code; return code;
} }
@ -643,7 +650,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
// restore the rows // restore the rows
pBlock->info.rows = backupRows; pBlock->info.rows = backupRows;
if (freeReader) { if (freeReader) {
metaReaderClear(&mr); pHandle->api.metaReaderFn.clearReader(&mr);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -677,6 +684,8 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData,
static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
STableScanInfo* pTableScanInfo = pOperator->info; STableScanInfo* pTableScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSDataBlock* pBlock = pTableScanInfo->pResBlock; SSDataBlock* pBlock = pTableScanInfo->pResBlock;
bool hasNext = false; bool hasNext = false;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -684,9 +693,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
while (true) { while (true) {
code = tsdbNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); code = pAPI->storeReader.storeReaderNextDataBlock(pTableScanInfo->base.dataReader, &hasNext);
if (code) { if (code) {
tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader);
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
@ -695,12 +704,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
} }
if (isTaskKilled(pTaskInfo)) { if (isTaskKilled(pTaskInfo)) {
tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader);
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
} }
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pTableScanInfo->base.dataReader);
break; break;
} }
@ -733,14 +742,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime; pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime;
// todo refactor
/*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/
/*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/
// pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA;
// pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid;
// pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey;
return pBlock; return pBlock;
} }
return NULL; return NULL;
@ -749,7 +750,8 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pTableScanInfo = pOperator->info; STableScanInfo* pTableScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
// The read handle is not initialized yet, since no qualified tables exists // The read handle is not initialized yet, since no qualified tables exists
if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) { if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
@ -772,7 +774,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo));
// do prepare for the next round table scan operation // do prepare for the next round table scan operation
tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
} }
} }
@ -780,7 +782,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->scanTimes < total) {
if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) { if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) {
prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0); prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0);
tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
} }
@ -798,7 +800,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
pTableScanInfo->base.scanFlag = MAIN_SCAN; pTableScanInfo->base.scanFlag = MAIN_SCAN;
qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo));
tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); pAPI->storeReader.storeReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
} }
} }
} }
@ -809,6 +811,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) {
static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STableScanInfo* pInfo = pOperator->info; STableScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
// scan table one by one sequentially // scan table one by one sequentially
if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
@ -836,11 +839,11 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable);
taosRUnLockLatch(&pTaskInfo->lock); taosRUnLockLatch(&pTaskInfo->lock);
tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1); pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, &tInfo, 1);
qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables,
pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo));
tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond);
pInfo->scanTimes = 0; pInfo->scanTimes = 0;
} }
} else { // scan table group by group sequentially } else { // scan table group by group sequentially
@ -855,8 +858,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
ASSERT(pInfo->base.dataReader == NULL); ASSERT(pInfo->base.dataReader == NULL);
int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, int32_t code = pAPI->storeReader.storeReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock,
(STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables); (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
@ -884,8 +887,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
STableKeyInfo* pList = NULL; STableKeyInfo* pList = NULL;
tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num);
tsdbSetTableList(pInfo->base.dataReader, pList, num); pAPI->storeReader.storeReaderSetTableList(pInfo->base.dataReader, pList, num);
tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); pAPI->storeReader.storeReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond);
pInfo->scanTimes = 0; pInfo->scanTimes = 0;
result = doGroupedTableScan(pOperator); result = doGroupedTableScan(pOperator);
@ -907,10 +910,10 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr
return 0; return 0;
} }
static void destroyTableScanBase(STableScanBase* pBase) { static void destroyTableScanBase(STableScanBase* pBase, SStoreDataReaderFn* pAPI) {
cleanupQueryTableDataCond(&pBase->cond); cleanupQueryTableDataCond(&pBase->cond);
tsdbReaderClose(pBase->dataReader); pAPI->storeReaderClose(pBase->dataReader);
pBase->dataReader = NULL; pBase->dataReader = NULL;
if (pBase->matchInfo.pList != NULL) { if (pBase->matchInfo.pList != NULL) {
@ -926,7 +929,7 @@ static void destroyTableScanOperatorInfo(void* param) {
STableScanInfo* pTableScanInfo = (STableScanInfo*)param; STableScanInfo* pTableScanInfo = (STableScanInfo*)param;
blockDataDestroy(pTableScanInfo->pResBlock); blockDataDestroy(pTableScanInfo->pResBlock);
taosHashCleanup(pTableScanInfo->pIgnoreTables); taosHashCleanup(pTableScanInfo->pIgnoreTables);
destroyTableScanBase(&pTableScanInfo->base); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->readerAPI);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
@ -1071,8 +1074,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6
pTableScanInfo->base.cond.endVersion = version; pTableScanInfo->base.cond.endVersion = version;
pTableScanInfo->scanTimes = 0; pTableScanInfo->scanTimes = 0;
pTableScanInfo->currentGroupId = -1; pTableScanInfo->currentGroupId = -1;
tsdbReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader);
qDebug("1");
pTableScanInfo->base.dataReader = NULL; pTableScanInfo->base.dataReader = NULL;
} }
@ -1088,11 +1090,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs}; cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs};
SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo; SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSDataBlock* pBlock = pTableScanInfo->pResBlock; SSDataBlock* pBlock = pTableScanInfo->pResBlock;
STsdbReader* pReader = NULL; STsdbReader* pReader = NULL;
int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, int32_t code = pAPI->storeReader.storeReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock,
(STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false, NULL); (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
@ -1100,7 +1103,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
} }
bool hasNext = false; bool hasNext = false;
code = tsdbNextDataBlock(pReader, &hasNext); code = pAPI->storeReader.storeReaderNextDataBlock(pReader, &hasNext);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
@ -1108,12 +1111,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
} }
if (hasNext) { if (hasNext) {
/*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL); /*SSDataBlock* p = */ pAPI->storeReader.storeReaderRetrieveDataBlock(pReader, NULL);
doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
} }
tsdbReaderClose(pReader); pAPI->storeReader.storeReaderClose(pReader);
qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64
", suid:%" PRIu64, ", suid:%" PRIu64,
pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid);
@ -1187,7 +1190,8 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
break; break;
} }
resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); ASSERT(0);
// resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion);
pInfo->pTableScanOp->status = OP_OPENED; pInfo->pTableScanOp->status = OP_OPENED;
return true; return true;
} }
@ -1245,8 +1249,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
*pRowIndex = 0; *pRowIndex = 0;
pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->readerAPI.storeReaderClose(pTableScanInfo->base.dataReader);
qDebug("2");
pTableScanInfo->base.dataReader = NULL; pTableScanInfo->base.dataReader = NULL;
return NULL; return NULL;
} }
@ -1291,8 +1294,8 @@ static int32_t getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
pKey->win.ekey = endTs; pKey->win.ekey = endTs;
pKey->groupId = groupId; pKey->groupId = groupId;
SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey); void* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey);
int32_t code = streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pKey, NULL, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pKey); SET_SESSION_WIN_KEY_INVALID(pKey);
} }
@ -1463,11 +1466,11 @@ static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pS
} }
if (pInfo->tbnameCalSup.pExprInfo) { if (pInfo->tbnameCalSup.pExprInfo) {
void* parTbname = NULL; void* parTbname = NULL;
streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname); pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname);
memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN); memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN);
varDataSetLen(tbname, strlen(varDataVal(tbname))); varDataSetLen(tbname, strlen(varDataVal(tbname)));
streamFreeVal(parTbname); pInfo->stateStore.streamStateFreeVal(parTbname);
} }
appendOneRowToStreamSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId, appendOneRowToStreamSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId,
tbname[0] == 0 ? NULL : tbname); tbname[0] == 0 ? NULL : tbname);
@ -1498,7 +1501,7 @@ static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else { } else {
appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup,
pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes); pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes, &pInfo->stateStore);
} }
} }
@ -1529,7 +1532,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; TSKEY* tsCol = (TSKEY*)pColDataInfo->pData;
bool tableInserted = updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid); bool tableInserted = pInfo->stateStore.updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid);
for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) {
SResultRowInfo dumyInfo; SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1; dumyInfo.cur.pageId = -1;
@ -1545,10 +1548,9 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
isClosed = isCloseWindow(&win, &pInfo->twAggSup); isClosed = isCloseWindow(&win, &pInfo->twAggSup);
} }
// must check update info first. // must check update info first.
bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]); bool update = pInfo->stateStore.updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]);
bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && bool closedWin = isClosed && isSignleIntervalWindow(pInfo) &&
isDeletedStreamWindow(&win, pBlock->info.id.groupId, isDeletedStreamWindow(&win, pBlock->info.id.groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore);
pInfo->pState, &pInfo->twAggSup);
if ((update || closedWin) && out) { if ((update || closedWin) && out) {
qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin);
uint64_t gpId = 0; uint64_t gpId = 0;
@ -1636,58 +1638,30 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
} }
static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SStreamScanInfo* pInfo = pOperator->info; SStreamScanInfo* pInfo = pOperator->info;
const char* id = GET_TASKID(pTaskInfo); const char* id = GET_TASKID(pTaskInfo);
qDebug("start to exec queue scan, %s", id); qDebug("start to exec queue scan, %s", id);
#if 0
if (pTaskInfo->streamInfo.submit.msgStr != NULL) {
if (pInfo->tqReader->msg.msgStr == NULL) {
SPackedData submit = pTaskInfo->streamInfo.submit;
if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) {
qError("submit msg messed up when initing stream submit block %p", submit.msgStr);
return NULL;
}
}
blockDataCleanup(pInfo->pRes);
SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
while (tqNextBlockImpl(pInfo->tqReader, NULL)) {
int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL);
if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) {
continue;
}
setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true);
if (pBlockInfo->rows > 0) {
return pInfo->pRes;
}
}
pInfo->tqReader->msg = (SPackedData){0};
pTaskInfo->streamInfo.submit = (SPackedData){0};
return NULL;
}
#endif
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
if (pResult && pResult->info.rows > 0) { if (pResult && pResult->info.rows > 0) {
qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, ASSERT(0);
pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, // qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64,
pInfo->tqReader->pWalReader->curVersion); // pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey,
// pInfo->tqReader->pWalReader->curVersion);
tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey);
return pResult; return pResult;
} }
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->base.dataReader); pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL; pTSInfo->base.dataReader = NULL;
qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1);
if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) {
return NULL; return NULL;
} }
@ -1697,11 +1671,14 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) {
while (1) { while (1) {
bool hasResult = tqNextBlockInWal(pInfo->tqReader, id); bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id);
SSDataBlock* pRes = pInfo->tqReader->pResBlock; ASSERT(0);
SSDataBlock* pRes = NULL;
struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader);
// curVersion move to next, so currentOffset = curVersion - 1 // curVersion move to next, so currentOffset = curVersion - 1
tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion - 1);
if (hasResult) { if (hasResult) {
qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows, qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows,
@ -1737,9 +1714,10 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S
SColumnInfoData* pDstStartCol = taosArrayGet(pDst->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pDstStartCol = taosArrayGet(pDst->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pDstEndCol = taosArrayGet(pDst->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pDstEndCol = taosArrayGet(pDst->pDataBlock, END_TS_COLUMN_INDEX);
SColumnInfoData* pDstUidCol = taosArrayGet(pDst->pDataBlock, UID_COLUMN_INDEX); SColumnInfoData* pDstUidCol = taosArrayGet(pDst->pDataBlock, UID_COLUMN_INDEX);
int32_t j = 0;
int32_t j = 0;
for (int32_t i = 0; i < rows; i++) { for (int32_t i = 0; i < rows; i++) {
if (taosHashGet(pReader->tbIdHash, &uidCol[i], sizeof(uint64_t))) { if (pInfo->readerFn.tqReaderIsQueriedTable(pReader, uidCol[i])) {
colDataSetVal(pDstStartCol, j, (const char*)&startCol[i], false); colDataSetVal(pDstStartCol, j, (const char*)&startCol[i], false);
colDataSetVal(pDstEndCol, j, (const char*)&endCol[i], false); colDataSetVal(pDstEndCol, j, (const char*)&endCol[i], false);
colDataSetVal(pDstUidCol, j, (const char*)&uidCol[i], false); colDataSetVal(pDstUidCol, j, (const char*)&uidCol[i], false);
@ -1750,6 +1728,7 @@ static int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, S
j++; j++;
} }
} }
uint32_t cap = pDst->info.capacity; uint32_t cap = pDst->info.capacity;
pDst->info = pSrc->info; pDst->info = pSrc->info;
pDst->info.rows = j; pDst->info.rows = j;
@ -1777,7 +1756,8 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) { static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) {
if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) {
pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); ASSERT(0);
// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version);
checkUpdateData(pInfo, true, pBlock, true); checkUpdateData(pInfo, true, pBlock, true);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey);
if (pInfo->pUpdateDataRes->info.rows > 0) { if (pInfo->pUpdateDataRes->info.rows > 0) {
@ -1794,21 +1774,21 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl
} }
} }
int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { //int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) {
int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); // int32_t len = updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo);
*pBuff = taosMemoryCalloc(1, len); // *pBuff = taosMemoryCalloc(1, len);
updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo); // updateInfoSerialize(*pBuff, len, pInfo->pUpdateInfo);
return len; // return len;
} //}
// other properties are recovered from the execution plan // other properties are recovered from the execution plan
void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo) { void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) {
if (!pBuff || len == 0) { if (!pBuff || len == 0) {
return; return;
} }
SUpdateInfo* pUpInfo = updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0); void* pUpInfo = pInfo->stateStore.updateInfoInit(0, TSDB_TIME_PRECISION_MILLI, 0);
int32_t code = updateInfoDeserialize(pBuff, len, pUpInfo); int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pInfo->pUpdateInfo = pUpInfo; pInfo->pUpdateInfo = pUpInfo;
} }
@ -1817,6 +1797,8 @@ void streamScanOperatorDeocde(void* pBuff, int32_t len, SStreamScanInfo* pInfo)
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not // NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SStreamScanInfo* pInfo = pOperator->info; SStreamScanInfo* pInfo = pOperator->info;
qDebug("stream scan started, %s", GET_TASKID(pTaskInfo)); qDebug("stream scan started, %s", GET_TASKID(pTaskInfo));
@ -1839,7 +1821,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2;
} }
tsdbReaderClose(pTSInfo->base.dataReader); pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL; pTSInfo->base.dataReader = NULL;
pInfo->pTableScanOp->status = OP_OPENED; pInfo->pTableScanOp->status = OP_OPENED;
@ -1902,7 +1884,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
calBlockTbName(pInfo, pInfo->pRecoverRes); calBlockTbName(pInfo, pInfo->pRecoverRes);
if (pInfo->pUpdateInfo) { if (pInfo->pUpdateInfo) {
if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) { if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) {
TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex);
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
} else { } else {
doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes);
@ -1919,7 +1901,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
} }
pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE;
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->base.dataReader); pAPI->storeReader.storeReaderClose(pTSInfo->base.dataReader);
pTSInfo->base.dataReader = NULL; pTSInfo->base.dataReader = NULL;
@ -1943,7 +1925,7 @@ FETCH_NEXT_BLOCK:
SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current);
SSDataBlock* pBlock = pPacked->pDataBlock; SSDataBlock* pBlock = pPacked->pDataBlock;
if (pBlock->info.parTbName[0]) { if (pBlock->info.parTbName[0]) {
streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); pAPI->stateStore.streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName);
} }
// TODO move into scan // TODO move into scan
@ -1951,7 +1933,8 @@ FETCH_NEXT_BLOCK:
pBlock->info.calWin.ekey = INT64_MAX; pBlock->info.calWin.ekey = INT64_MAX;
pBlock->info.dataLoad = 1; pBlock->info.dataLoad = 1;
if (pInfo->pUpdateInfo) { if (pInfo->pUpdateInfo) {
pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version); ASSERT(0);
// pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version);
} }
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
switch (pBlock->info.type) { switch (pBlock->info.type) {
@ -1964,7 +1947,7 @@ FETCH_NEXT_BLOCK:
copyDataBlock(pInfo->pUpdateRes, pBlock); copyDataBlock(pInfo->pUpdateRes, pBlock);
pInfo->updateResIndex = 0; pInfo->updateResIndex = 0;
prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); pAPI->stateStore.updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
} break; } break;
case STREAM_DELETE_DATA: { case STREAM_DELETE_DATA: {
printDataBlock(pBlock, "stream scan delete recv"); printDataBlock(pBlock, "stream scan delete recv");
@ -2081,9 +2064,9 @@ FETCH_NEXT_BLOCK:
NEXT_SUBMIT_BLK: NEXT_SUBMIT_BLK:
while (1) { while (1) {
if (pInfo->tqReader->msg.msgStr == NULL) { if (pInfo->readerFn.tqReaderCurrentBlockConsumed(pInfo->tqReader)) {
if (pInfo->validBlockIndex >= totalBlocks) { if (pInfo->validBlockIndex >= totalBlocks) {
updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); pAPI->stateStore.updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
doClearBufferedBlocks(pInfo); doClearBufferedBlocks(pInfo);
qDebug("stream scan return empty, all %d submit blocks consumed, %s", totalBlocks, id); qDebug("stream scan return empty, all %d submit blocks consumed, %s", totalBlocks, id);
@ -2094,7 +2077,7 @@ FETCH_NEXT_BLOCK:
SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current);
qDebug("set %d/%d as the input submit block, %s", current, totalBlocks, id); qDebug("set %d/%d as the input submit block, %s", current, totalBlocks, id);
if (tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) {
qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current, totalBlocks, id); qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current, totalBlocks, id);
continue; continue;
} }
@ -2102,13 +2085,14 @@ FETCH_NEXT_BLOCK:
blockDataCleanup(pBlock); blockDataCleanup(pBlock);
while (tqNextBlockImpl(pInfo->tqReader, id)) { while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) {
int32_t code = tqRetrieveDataBlock(pInfo->tqReader, id); SSDataBlock* pRes = NULL;
if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id);
if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) {
continue; continue;
} }
setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, false); setBlockIntoRes(pInfo, pRes, false);
if (pInfo->pCreateTbRes->info.rows > 0) { if (pInfo->pCreateTbRes->info.rows > 0) {
pInfo->scanMode = STREAM_SCAN_FROM_RES; pInfo->scanMode = STREAM_SCAN_FROM_RES;
@ -2166,7 +2150,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableListInfo) {
static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not // NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SStreamRawScanInfo* pInfo = pOperator->info; SStreamRawScanInfo* pInfo = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta
@ -2176,20 +2162,20 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
bool hasNext = false; bool hasNext = false;
if (pInfo->dataReader) { if (pInfo->dataReader) {
code = tsdbNextDataBlock(pInfo->dataReader, &hasNext); code = pAPI->storeReader.storeReaderNextDataBlock(pInfo->dataReader, &hasNext);
if (code) { if (code) {
tsdbReleaseDataBlock(pInfo->dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader);
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
} }
if (pInfo->dataReader && hasNext) { if (pInfo->dataReader && hasNext) {
if (isTaskKilled(pTaskInfo)) { if (isTaskKilled(pTaskInfo)) {
tsdbReleaseDataBlock(pInfo->dataReader); pAPI->storeReader.storeReaderReleaseDataBlock(pInfo->dataReader);
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
} }
SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); SSDataBlock* pBlock = pAPI->storeReader.storeReaderRetrieveDataBlock(pInfo->dataReader, NULL);
if (pBlock == NULL) { if (pBlock == NULL) {
T_LONG_JMP(pTaskInfo->env, terrno); T_LONG_JMP(pTaskInfo->env, terrno);
} }
@ -2199,7 +2185,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
return pBlock; return pBlock;
} }
SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); SMetaTableInfo mtInfo = pAPI->snapshotFn.storeSSGetTableInfo(pInfo->sContext);
STqOffsetVal offset = {0}; STqOffsetVal offset = {0};
if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal
qDebug("tmqsnap read snapshot done, change to get data from wal"); qDebug("tmqsnap read snapshot done, change to get data from wal");
@ -2217,7 +2203,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
int32_t dataLen = 0; int32_t dataLen = 0;
int16_t type = 0; int16_t type = 0;
int64_t uid = 0; int64_t uid = 0;
if (getMetafromSnapShot(sContext, &data, &dataLen, &type, &uid) < 0) { if (pAPI->snapshotFn.storeSSGetMetaInfo(sContext, &data, &dataLen, &type, &uid) < 0) {
qError("tmqsnap getMetafromSnapShot error"); qError("tmqsnap getMetafromSnapShot error");
taosMemoryFreeClear(data); taosMemoryFreeClear(data);
return NULL; return NULL;
@ -2241,8 +2227,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
static void destroyRawScanOperatorInfo(void* param) { static void destroyRawScanOperatorInfo(void* param) {
SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param;
tsdbReaderClose(pRawScan->dataReader); pRawScan->pAPI->storeReader.storeReaderClose(pRawScan->dataReader);
destroySnapContext(pRawScan->sContext); pRawScan->pAPI->snapshotFn.storeDestroySnapshot(pRawScan->sContext);
tableListDestroy(pRawScan->pTableListInfo); tableListDestroy(pRawScan->pTableListInfo);
taosMemoryFree(pRawScan); taosMemoryFree(pRawScan);
} }
@ -2290,7 +2276,7 @@ static void destroyStreamScanOperatorInfo(void* param) {
} }
if (pStreamScan->tqReader) { if (pStreamScan->tqReader) {
tqCloseReader(pStreamScan->tqReader); pStreamScan->readerFn.tqReaderClose(pStreamScan->tqReader);
} }
if (pStreamScan->matchInfo.pList) { if (pStreamScan->matchInfo.pList) {
taosArrayDestroy(pStreamScan->matchInfo.pList); taosArrayDestroy(pStreamScan->matchInfo.pList);
@ -2303,7 +2289,7 @@ static void destroyStreamScanOperatorInfo(void* param) {
cleanupExprSupp(&pStreamScan->tbnameCalSup); cleanupExprSupp(&pStreamScan->tbnameCalSup);
cleanupExprSupp(&pStreamScan->tagCalSup); cleanupExprSupp(&pStreamScan->tagCalSup);
updateInfoDestroy(pStreamScan->pUpdateInfo); pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo);
blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pRes);
blockDataDestroy(pStreamScan->pUpdateRes); blockDataDestroy(pStreamScan->pUpdateRes);
blockDataDestroy(pStreamScan->pPullDataRes); blockDataDestroy(pStreamScan->pPullDataRes);
@ -2319,6 +2305,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
SArray* pColIds = NULL; SArray* pColIds = NULL;
SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -2408,7 +2395,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
if (pHandle->initTqReader) { if (pHandle->initTqReader) {
ASSERT(pHandle->tqReader == NULL); ASSERT(pHandle->tqReader == NULL);
pInfo->tqReader = tqReaderOpen(pHandle->vnode); pInfo->tqReader = pAPI->tqReaderFn.tqReaderOpen(pHandle->vnode);
ASSERT(pInfo->tqReader); ASSERT(pInfo->tqReader);
} else { } else {
ASSERT(pHandle->tqReader); ASSERT(pHandle->tqReader);
@ -2418,7 +2405,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo->pUpdateInfo = NULL; pInfo->pUpdateInfo = NULL;
pInfo->pTableScanOp = pTableScanOp; pInfo->pTableScanOp = pTableScanOp;
if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) { if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) {
streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1); pAPI->stateStore.streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1);
} }
pInfo->readHandle = *pHandle; pInfo->readHandle = *pHandle;
@ -2427,9 +2414,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); blockDataEnsureCapacity(pInfo->pCreateTbRes, 8);
// set the extract column id to streamHandle // set the extract column id to streamHandle
tqReaderSetColIdList(pInfo->tqReader, pColIds); pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds);
SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo);
code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); code = pAPI->tqReaderFn.tqReaderSetTargetTableList(pInfo->tqReader, tableIdList);
if (code != 0) { if (code != 0) {
taosArrayDestroy(tableIdList); taosArrayDestroy(tableIdList);
goto _error; goto _error;
@ -2475,8 +2462,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
if (pTaskInfo->streamInfo.pState) { if (pTaskInfo->streamInfo.pState) {
void* buff = NULL; void* buff = NULL;
int32_t len = 0; int32_t len = 0;
streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len); pAPI->stateStore.streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len);
streamScanOperatorDeocde(buff, len, pInfo); streamScanOperatorDecode(buff, len, pInfo);
} }
setOperatorInfo(pOperator, STREAM_SCAN_OP_NAME, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, setOperatorInfo(pOperator, STREAM_SCAN_OP_NAME, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo,
@ -2502,18 +2489,18 @@ _error:
return NULL; return NULL;
} }
static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr) { static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr, SStorageAPI* pAPI) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
STagScanInfo* pInfo = pOperator->info; STagScanInfo* pInfo = pOperator->info;
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0];
STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos);
int32_t code = metaGetTableEntryByUid(mr, item->uid); int32_t code = pAPI->metaReaderFn.getTableEntryByUid(mr, item->uid);
tDecoderClear(&(*mr).coder); tDecoderClear(&(*mr).coder);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno),
GET_TASKID(pTaskInfo)); GET_TASKID(pTaskInfo));
metaReaderClear(mr); pAPI->metaReaderFn.clearReader(mr);
T_LONG_JMP(pTaskInfo->env, terrno); T_LONG_JMP(pTaskInfo->env, terrno);
} }
@ -2528,7 +2515,7 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes,
} else { // it is a tag value } else { // it is a tag value
STagVal val = {0}; STagVal val = {0};
val.cid = pExprInfo[j].base.pParam[0].pCol->colId; val.cid = pExprInfo[j].base.pParam[0].pCol->colId;
const char* p = metaGetTableTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val); const char* p = pAPI->metaFn.extractTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val);
char* data = NULL; char* data = NULL;
if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) {
@ -2553,6 +2540,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
} }
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
STagScanInfo* pInfo = pOperator->info; STagScanInfo* pInfo = pOperator->info;
SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0];
@ -2568,10 +2556,10 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
char str[512] = {0}; char str[512] = {0};
int32_t count = 0; int32_t count = 0;
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pInfo->readHandle.meta, 0); pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0);
while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) {
doTagScanOneTable(pOperator, pRes, count, &mr); doTagScanOneTable(pOperator, pRes, count, &mr, &pTaskInfo->storageAPI);
++count; ++count;
if (++pInfo->curPos >= size) { if (++pInfo->curPos >= size) {
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
@ -2589,7 +2577,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
} }
} }
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
// qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
@ -2661,6 +2649,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
SOperatorInfo* pOperator = source->pOperator; SOperatorInfo* pOperator = source->pOperator;
STableMergeScanInfo* pInfo = pOperator->info; STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
int32_t readIdx = source->readerIdx; int32_t readIdx = source->readerIdx;
SSDataBlock* pBlock = source->inputBlock; SSDataBlock* pBlock = source->inputBlock;
int32_t code = 0; int32_t code = 0;
@ -2672,7 +2662,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
SReadHandle* pHandle = &pInfo->base.readHandle; SReadHandle* pHandle = &pInfo->base.readHandle;
if (NULL == source->dataReader || !source->multiReader) { if (NULL == source->dataReader || !source->multiReader) {
code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false, NULL); code = pAPI->storeReader.storeReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, (void**)&source->dataReader, GET_TASKID(pTaskInfo), false, NULL);
if (code != 0) { if (code != 0) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
@ -2684,9 +2674,9 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
qTrace("tsdb/read-table-data: %p, enter next reader", reader); qTrace("tsdb/read-table-data: %p, enter next reader", reader);
while (true) { while (true) {
code = tsdbNextDataBlock(reader, &hasNext); code = pAPI->storeReader.storeReaderNextDataBlock(reader, &hasNext);
if (code != 0) { if (code != 0) {
tsdbReleaseDataBlock(reader); pAPI->storeReader.storeReaderReleaseDataBlock(reader);
pInfo->base.dataReader = NULL; pInfo->base.dataReader = NULL;
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
@ -2696,7 +2686,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
} }
if (isTaskKilled(pTaskInfo)) { if (isTaskKilled(pTaskInfo)) {
tsdbReleaseDataBlock(reader); pAPI->storeReader.storeReaderReleaseDataBlock(reader);
pInfo->base.dataReader = NULL; pInfo->base.dataReader = NULL;
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
} }
@ -2736,7 +2726,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
qTrace("tsdb/read-table-data: %p, close reader", reader); qTrace("tsdb/read-table-data: %p, close reader", reader);
if (!source->multiReader) { if (!source->multiReader) {
tsdbReaderClose(pInfo->base.dataReader); pAPI->storeReader.storeReaderClose(pInfo->base.dataReader);
source->dataReader = NULL; source->dataReader = NULL;
} }
pInfo->base.dataReader = NULL; pInfo->base.dataReader = NULL;
@ -2744,7 +2734,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
} }
if (!source->multiReader) { if (!source->multiReader) {
tsdbReaderClose(pInfo->base.dataReader); pAPI->storeReader.storeReaderClose(pInfo->base.dataReader);
source->dataReader = NULL; source->dataReader = NULL;
} }
pInfo->base.dataReader = NULL; pInfo->base.dataReader = NULL;
@ -2849,6 +2839,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
STableMergeScanInfo* pInfo = pOperator->info; STableMergeScanInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
int32_t numOfTable = taosArrayGetSize(pInfo->queryConds); int32_t numOfTable = taosArrayGetSize(pInfo->queryConds);
@ -2862,7 +2853,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
for (int32_t i = 0; i < numOfTable; ++i) { for (int32_t i = 0; i < numOfTable; ++i) {
STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i);
blockDataDestroy(param->inputBlock); blockDataDestroy(param->inputBlock);
tsdbReaderClose(param->dataReader); pAPI->storeReader.storeReaderClose(param->dataReader);
param->dataReader = NULL; param->dataReader = NULL;
} }
taosArrayClear(pInfo->sortSourceParams); taosArrayClear(pInfo->sortSourceParams);
@ -2974,11 +2965,11 @@ void destroyTableMergeScanOperatorInfo(void* param) {
for (int32_t i = 0; i < numOfTable; i++) { for (int32_t i = 0; i < numOfTable; i++) {
STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i); STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i);
blockDataDestroy(p->inputBlock); blockDataDestroy(p->inputBlock);
tsdbReaderClose(p->dataReader); pTableScanInfo->base.readerAPI.storeReaderClose(p->dataReader);
p->dataReader = NULL; p->dataReader = NULL;
} }
tsdbReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.readerAPI.storeReaderClose(pTableScanInfo->base.dataReader);
pTableScanInfo->base.dataReader = NULL; pTableScanInfo->base.dataReader = NULL;
taosArrayDestroy(pTableScanInfo->sortSourceParams); taosArrayDestroy(pTableScanInfo->sortSourceParams);
@ -2991,7 +2982,7 @@ void destroyTableMergeScanOperatorInfo(void* param) {
} }
taosArrayDestroy(pTableScanInfo->queryConds); taosArrayDestroy(pTableScanInfo->queryConds);
destroyTableScanBase(&pTableScanInfo->base); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI);
pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock);
pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock);
@ -3359,9 +3350,11 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount
const char* db = NULL; const char* db = NULL;
int32_t vgId = 0; int32_t vgId = 0;
char dbName[TSDB_DB_NAME_LEN] = {0}; char dbName[TSDB_DB_NAME_LEN] = {0};
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
// get dbname // get dbname
vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId);
SName sn = {0}; SName sn = {0};
tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&sn, dbName); tNameGetDbName(&sn, dbName);
@ -3376,10 +3369,13 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount
static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
if (pSupp->groupByStbName) { if (pSupp->groupByStbName) {
if (pInfo->stbUidList == NULL) { if (pInfo->stbUidList == NULL) {
pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t));
if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { if (pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, 0, pInfo->stbUidList, TSDB_SUPER_TABLE) < 0) {
qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr());
} }
} }
@ -3398,7 +3394,9 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca
} else { } else {
uint64_t groupId = calcGroupId(dbName, strlen(dbName)); uint64_t groupId = calcGroupId(dbName, strlen(dbName));
pRes->info.id.groupId = groupId; pRes->info.id.groupId = groupId;
int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta);
int64_t dbTableCount = 0;
pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.meta, &dbTableCount);
fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes);
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
} }
@ -3406,19 +3404,25 @@ static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountSca
static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
if (strlen(pSupp->dbNameFilter) != 0) { if (strlen(pSupp->dbNameFilter) != 0) {
if (strlen(pSupp->stbNameFilter) != 0) { if (strlen(pSupp->stbNameFilter) != 0) {
tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter); tb_uid_t uid = 0;
pAPI->metaFn.getTableUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter, &uid);
SMetaStbStats stats = {0}; SMetaStbStats stats = {0};
metaGetStbStats(pInfo->readHandle.meta, uid, &stats); ASSERT(0);
// metaGetStbStats(pInfo->readHandle.meta, uid, &stats);
int64_t ctbNum = stats.ctbNum; int64_t ctbNum = stats.ctbNum;
fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes);
} else { } else {
int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta);
fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
} }
} else { } else {
int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); int64_t tbNumVnode = 0;//metaGetTbNum(pInfo->readHandle.meta);
pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode);
fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
} }
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
@ -3433,7 +3437,8 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S
uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
pRes->info.id.groupId = groupId; pRes->info.id.groupId = groupId;
int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); int64_t ntbNum = 0;//metaGetNtbNum(pInfo->readHandle.meta);
ASSERT(0);
if (ntbNum != 0) { if (ntbNum != 0) {
fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes);
} }
@ -3442,7 +3447,8 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S
static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) {
char stbName[TSDB_TABLE_NAME_LEN] = {0}; char stbName[TSDB_TABLE_NAME_LEN] = {0};
metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); ASSERT(0);
// metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName);
char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
if (pSupp->groupByDbName) { if (pSupp->groupByDbName) {
@ -3455,7 +3461,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S
pRes->info.id.groupId = groupId; pRes->info.id.groupId = groupId;
SMetaStbStats stats = {0}; SMetaStbStats stats = {0};
metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); // metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats);
int64_t ctbNum = stats.ctbNum; int64_t ctbNum = stats.ctbNum;
fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes);

191
source/libs/executor/src/sysscanoperator.c
View File

@ -21,19 +21,19 @@
#include "querynodes.h" #include "querynodes.h"
#include "systable.h" #include "systable.h"
#include "tname.h" #include "tname.h"
#include "ttime.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tmsg.h" #include "tmsg.h"
#include "operator.h"
#include "query.h" #include "query.h"
#include "querytask.h"
#include "storageapi.h"
#include "tcompare.h" #include "tcompare.h"
#include "thash.h" #include "thash.h"
#include "ttypes.h" #include "ttypes.h"
#include "vnode.h" #include "tsdstorage.h"
#include "operator.h" #include "index.h"
#include "querytask.h"
typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype);
typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result);
@ -72,6 +72,7 @@ typedef struct SSysTableScanInfo {
SLoadRemoteDataInfo loadInfo; SLoadRemoteDataInfo loadInfo;
SLimitInfo limitInfo; SLimitInfo limitInfo;
int32_t tbnameSlotId; int32_t tbnameSlotId;
SStorageAPI* pAPI;
} SSysTableScanInfo; } SSysTableScanInfo;
typedef struct { typedef struct {
@ -156,7 +157,8 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) {
void* pVnode = ((SSTabFltArg*)arg)->pVnode; void* pVnode = ((SSTabFltArg*)arg)->pVnode;
const char* db = NULL; const char* db = NULL;
vnodeGetInfo(pVnode, &db, NULL); ASSERT(0);
// pAPI->metaFn.storeGetBasicInfo(pVnode, &db, NULL);
SName sn = {0}; SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@ -183,7 +185,8 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) {
void* pVnode = ((SSTabFltArg*)arg)->pVnode; void* pVnode = ((SSTabFltArg*)arg)->pVnode;
int64_t vgId = 0; int64_t vgId = 0;
vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); ASSERT(0);
// pAPI->metaFn.storeGetBasicInfo(pVnode, NULL, (int32_t*)&vgId);
SOperatorNode* pOper = (SOperatorNode*)pNode; SOperatorNode* pOper = (SOperatorNode*)pNode;
SValueNode* pVal = (SValueNode*)pOper->pRight; SValueNode* pVal = (SValueNode*)pOper->pRight;
@ -237,8 +240,10 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) {
.equal = equal, .equal = equal,
.filterFunc = func}; .filterFunc = func};
int32_t ret = metaFilterCreateTime(pMeta, &param, result); ASSERT(0);
return ret; return 0;
// int32_t ret = metaFilterCreateTime(pMeta, &param, result);
// return ret;
} }
int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) {
@ -431,8 +436,9 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) {
} }
static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
qDebug("sysTableScanUserCols get cols start"); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info; SSysTableScanInfo* pInfo = pOperator->info;
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
@ -446,7 +452,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
const char* db = NULL; const char* db = NULL;
int32_t vgId = 0; int32_t vgId = 0;
vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId);
SName sn = {0}; SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@ -461,18 +467,18 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
STR_TO_VARSTR(tableName, pInfo->req.filterTb); STR_TO_VARSTR(tableName, pInfo->req.filterTb);
SMetaReader smrTable = {0}; SMetaReader smrTable = {0};
metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0);
int32_t code = metaGetTableEntryByName(&smrTable, pInfo->req.filterTb); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByName, therefore, return directly // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
metaReaderClear(&smrTable); pAPI->metaReaderFn.clearReader(&smrTable);
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0; pInfo->loadInfo.totalRows = 0;
return NULL; return NULL;
} }
if (smrTable.me.type == TSDB_SUPER_TABLE) { if (smrTable.me.type == TSDB_SUPER_TABLE) {
metaReaderClear(&smrTable); pAPI->metaReaderFn.clearReader(&smrTable);
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0; pInfo->loadInfo.totalRows = 0;
return NULL; return NULL;
@ -480,12 +486,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
if (smrTable.me.type == TSDB_CHILD_TABLE) { if (smrTable.me.type == TSDB_CHILD_TABLE) {
int64_t suid = smrTable.me.ctbEntry.suid; int64_t suid = smrTable.me.ctbEntry.suid;
metaReaderClear(&smrTable); pAPI->metaReaderFn.clearReader(&smrTable);
metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.meta, 0);
code = metaGetTableEntryByUid(&smrTable, suid); code = pAPI->metaReaderFn.getTableEntryByUid(&smrTable, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByName, therefore, return directly // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
metaReaderClear(&smrTable); pAPI->metaReaderFn.clearReader(&smrTable);
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0; pInfo->loadInfo.totalRows = 0;
return NULL; return NULL;
@ -503,7 +509,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
} }
sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName);
metaReaderClear(&smrTable); pAPI->metaReaderFn.clearReader(&smrTable);
if (numOfRows > 0) { if (numOfRows > 0) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
@ -517,7 +523,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
int32_t ret = 0; int32_t ret = 0;
if (pInfo->pCur == NULL) { if (pInfo->pCur == NULL) {
pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta);
} }
if (pInfo->pSchema == NULL) { if (pInfo->pSchema == NULL) {
@ -535,8 +541,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
int32_t restore = pInfo->restore; int32_t restore = pInfo->restore;
pInfo->restore = false; pInfo->restore = false;
while (restore || ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) {
if (restore) restore = false; while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) {
if (restore) {
restore = false;
}
char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
@ -560,12 +570,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
schemaRow = *(SSchemaWrapper**)schema; schemaRow = *(SSchemaWrapper**)schema;
} else { } else {
SMetaReader smrSuperTable = {0}; SMetaReader smrSuperTable = {0};
metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0);
int code = metaGetTableEntryByUid(&smrSuperTable, suid); int code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByName, therefore, return directly // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code); qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code);
metaReaderClear(&smrSuperTable); pAPI->metaReaderFn.clearReader(&smrSuperTable);
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0; pInfo->loadInfo.totalRows = 0;
return NULL; return NULL;
@ -573,7 +583,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow); SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow);
taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES);
schemaRow = schemaWrapper; schemaRow = schemaWrapper;
metaReaderClear(&smrSuperTable); pAPI->metaReaderFn.clearReader(&smrSuperTable);
} }
} else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) {
qDebug("sysTableScanUserCols cursor get normal table"); qDebug("sysTableScanUserCols cursor get normal table");
@ -605,7 +615,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
if (ret != 0) { if (ret != 0) {
metaCloseTbCursor(pInfo->pCur); pAPI->metaFn.closeMetaCursor(pInfo->pCur);
pInfo->pCur = NULL; pInfo->pCur = NULL;
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
} }
@ -618,6 +628,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info; SSysTableScanInfo* pInfo = pOperator->info;
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
@ -631,7 +643,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
const char* db = NULL; const char* db = NULL;
int32_t vgId = 0; int32_t vgId = 0;
vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId);
SName sn = {0}; SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@ -647,37 +659,37 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
STR_TO_VARSTR(tableName, condTableName); STR_TO_VARSTR(tableName, condTableName);
SMetaReader smrChildTable = {0}; SMetaReader smrChildTable = {0};
metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); pAPI->metaReaderFn.initReader(&smrChildTable, pInfo->readHandle.meta, 0);
int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrChildTable, condTableName);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByName, therefore, return directly // terrno has been set by pAPI->metaReaderFn.getTableEntryByName, therefore, return directly
metaReaderClear(&smrChildTable); pAPI->metaReaderFn.clearReader(&smrChildTable);
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0; pInfo->loadInfo.totalRows = 0;
return NULL; return NULL;
} }
if (smrChildTable.me.type != TSDB_CHILD_TABLE) { if (smrChildTable.me.type != TSDB_CHILD_TABLE) {
metaReaderClear(&smrChildTable); pAPI->metaReaderFn.clearReader(&smrChildTable);
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0; pInfo->loadInfo.totalRows = 0;
return NULL; return NULL;
} }
SMetaReader smrSuperTable = {0}; SMetaReader smrSuperTable = {0};
metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK);
code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
// terrno has been set by metaGetTableEntryByUid // terrno has been set by pAPI->metaReaderFn.getTableEntryByUid
metaReaderClear(&smrSuperTable); pAPI->metaReaderFn.clearReader(&smrSuperTable);
metaReaderClear(&smrChildTable); pAPI->metaReaderFn.clearReader(&smrChildTable);
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
return NULL; return NULL;
} }
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
metaReaderClear(&smrSuperTable); pAPI->metaReaderFn.clearReader(&smrSuperTable);
metaReaderClear(&smrChildTable); pAPI->metaReaderFn.clearReader(&smrChildTable);
if (numOfRows > 0) { if (numOfRows > 0) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
@ -691,11 +703,11 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
int32_t ret = 0; int32_t ret = 0;
if (pInfo->pCur == NULL) { if (pInfo->pCur == NULL) {
pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta);
} }
bool blockFull = false; bool blockFull = false;
while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) {
continue; continue;
} }
@ -704,27 +716,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name);
SMetaReader smrSuperTable = {0}; SMetaReader smrSuperTable = {0};
metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.meta, 0);
uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno),
GET_TASKID(pTaskInfo)); GET_TASKID(pTaskInfo));
metaReaderClear(&smrSuperTable); pAPI->metaReaderFn.clearReader(&smrSuperTable);
metaCloseTbCursor(pInfo->pCur); pAPI->metaFn.closeMetaCursor(pInfo->pCur);
pInfo->pCur = NULL; pInfo->pCur = NULL;
T_LONG_JMP(pTaskInfo->env, terrno); T_LONG_JMP(pTaskInfo->env, terrno);
} }
if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) { if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) {
metaTbCursorPrev(pInfo->pCur, TSDB_TABLE_MAX); pAPI->metaFn.cursorPrev(pInfo->pCur, TSDB_TABLE_MAX);
blockFull = true; blockFull = true;
} else { } else {
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows,
dataBlock); dataBlock);
} }
metaReaderClear(&smrSuperTable); pAPI->metaReaderFn.clearReader(&smrSuperTable);
if (blockFull || numOfRows >= pOperator->resultInfo.capacity) { if (blockFull || numOfRows >= pOperator->resultInfo.capacity) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
@ -745,7 +757,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
if (ret != 0) { if (ret != 0) {
metaCloseTbCursor(pInfo->pCur); pAPI->metaFn.closeMetaCursor(pInfo->pCur);
pInfo->pCur = NULL; pInfo->pCur = NULL;
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
} }
@ -1090,6 +1102,8 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) {
static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info; SSysTableScanInfo* pInfo = pOperator->info;
SSysTableIndex* pIdx = pInfo->pIdx; SSysTableIndex* pIdx = pInfo->pIdx;
@ -1100,7 +1114,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
const char* db = NULL; const char* db = NULL;
int32_t vgId = 0; int32_t vgId = 0;
vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId);
SName sn = {0}; SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@ -1118,10 +1132,10 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
tb_uid_t* uid = taosArrayGet(pIdx->uids, i); tb_uid_t* uid = taosArrayGet(pIdx->uids, i);
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pInfo->readHandle.meta, 0); pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, 0);
ret = metaGetTableEntryByUid(&mr, *uid); ret = pAPI->metaReaderFn.getTableEntryByUid(&mr, *uid);
if (ret < 0) { if (ret < 0) {
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
continue; continue;
} }
STR_TO_VARSTR(n, mr.me.name); STR_TO_VARSTR(n, mr.me.name);
@ -1146,15 +1160,15 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
SMetaReader mr1 = {0}; SMetaReader mr1 = {0};
metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); pAPI->metaReaderFn.initReader(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK);
int64_t suid = mr.me.ctbEntry.suid; int64_t suid = mr.me.ctbEntry.suid;
int32_t code = metaGetTableEntryByUid(&mr1, suid); int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr1, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name,
suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
metaReaderClear(&mr1); pAPI->metaReaderFn.clearReader(&mr1);
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
T_LONG_JMP(pTaskInfo->env, terrno); T_LONG_JMP(pTaskInfo->env, terrno);
} }
pColInfoData = taosArrayGet(p->pDataBlock, 3); pColInfoData = taosArrayGet(p->pDataBlock, 3);
@ -1164,7 +1178,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
STR_TO_VARSTR(n, mr1.me.name); STR_TO_VARSTR(n, mr1.me.name);
pColInfoData = taosArrayGet(p->pDataBlock, 4); pColInfoData = taosArrayGet(p->pDataBlock, 4);
colDataSetVal(pColInfoData, numOfRows, n, false); colDataSetVal(pColInfoData, numOfRows, n, false);
metaReaderClear(&mr1); pAPI->metaReaderFn.clearReader(&mr1);
// table comment // table comment
pColInfoData = taosArrayGet(p->pDataBlock, 8); pColInfoData = taosArrayGet(p->pDataBlock, 8);
@ -1229,7 +1243,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
// impl later // impl later
} }
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
pColInfoData = taosArrayGet(p->pDataBlock, 9); pColInfoData = taosArrayGet(p->pDataBlock, 9);
colDataSetVal(pColInfoData, numOfRows, n, false); colDataSetVal(pColInfoData, numOfRows, n, false);
@ -1275,10 +1289,11 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SSysTableScanInfo* pInfo = pOperator->info; SSysTableScanInfo* pInfo = pOperator->info;
if (pInfo->pCur == NULL) { if (pInfo->pCur == NULL) {
pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); pInfo->pCur = pAPI->metaFn.openMetaCursor(pInfo->readHandle.meta);
} }
blockDataCleanup(pInfo->pRes); blockDataCleanup(pInfo->pRes);
@ -1286,7 +1301,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
const char* db = NULL; const char* db = NULL;
int32_t vgId = 0; int32_t vgId = 0;
vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); pAPI->metaFn.storeGetBasicInfo(pInfo->readHandle.vnode, &db, &vgId);
SName sn = {0}; SName sn = {0};
char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
@ -1301,7 +1316,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
int32_t ret = 0; int32_t ret = 0;
while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) {
STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); STR_TO_VARSTR(n, pInfo->pCur->mr.me.name);
// table name // table name
@ -1324,15 +1339,15 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.meta, META_READER_NOLOCK);
uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
int32_t code = metaGetTableEntryByUid(&mr, suid); int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name,
suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
metaCloseTbCursor(pInfo->pCur); pAPI->metaFn.closeMetaCursor(pInfo->pCur);
pInfo->pCur = NULL; pInfo->pCur = NULL;
T_LONG_JMP(pTaskInfo->env, terrno); T_LONG_JMP(pTaskInfo->env, terrno);
} }
@ -1345,7 +1360,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
STR_TO_VARSTR(n, mr.me.name); STR_TO_VARSTR(n, mr.me.name);
pColInfoData = taosArrayGet(p->pDataBlock, 4); pColInfoData = taosArrayGet(p->pDataBlock, 4);
colDataSetVal(pColInfoData, numOfRows, n, false); colDataSetVal(pColInfoData, numOfRows, n, false);
metaReaderClear(&mr); pAPI->metaReaderFn.clearReader(&mr);
// table comment // table comment
pColInfoData = taosArrayGet(p->pDataBlock, 8); pColInfoData = taosArrayGet(p->pDataBlock, 8);
@ -1442,7 +1457,8 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
// todo temporarily free the cursor here, the true reason why the free is not valid needs to be found // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found
if (ret != 0) { if (ret != 0) {
metaCloseTbCursor(pInfo->pCur); pAPI->metaFn.closeMetaCursor(pInfo->pCur);
pAPI->metaFn.closeMetaCursor(pInfo->pCur);
pInfo->pCur = NULL; pInfo->pCur = NULL;
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
} }
@ -1708,7 +1724,7 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode,
const char* pUser, SExecTaskInfo* pTaskInfo) { const char* pUser, SExecTaskInfo* pTaskInfo) {
int32_t code = TDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -1798,7 +1814,7 @@ void destroySysScanOperator(void* param) {
if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 ||
strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 ||
strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) {
metaCloseTbCursor(pInfo->pCur); pInfo->pAPI->metaFn.closeMetaCursor(pInfo->pCur);
pInfo->pCur = NULL; pInfo->pCur = NULL;
} }
if (pInfo->pIdx) { if (pInfo->pIdx) {
@ -2124,15 +2140,15 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) {
return -1; return -1;
} }
static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { static int32_t doGetTableRowSize(SReadHandle *pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) {
*rowLen = 0; *rowLen = 0;
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pMeta, 0); pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, 0);
int32_t code = metaGetTableEntryByUid(&mr, uid); int32_t code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, uid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr);
metaReaderClear(&mr); pHandle->api.metaReaderFn.clearReader(&mr);
return terrno; return terrno;
} }
@ -2144,10 +2160,10 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
} else if (mr.me.type == TSDB_CHILD_TABLE) { } else if (mr.me.type == TSDB_CHILD_TABLE) {
uint64_t suid = mr.me.ctbEntry.suid; uint64_t suid = mr.me.ctbEntry.suid;
tDecoderClear(&mr.coder); tDecoderClear(&mr.coder);
code = metaGetTableEntryByUid(&mr, suid); code = pHandle->api.metaReaderFn.getTableEntryByUid(&mr, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr);
metaReaderClear(&mr); pHandle->api.metaReaderFn.clearReader(&mr);
return terrno; return terrno;
} }
@ -2163,7 +2179,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
} }
} }
metaReaderClear(&mr); pHandle->api.metaReaderFn.clearReader(&mr);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2174,16 +2190,17 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
SBlockDistInfo* pBlockScanInfo = pOperator->info; SBlockDistInfo* pBlockScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN};
int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, int32_t code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid,
(int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code); T_LONG_JMP(pTaskInfo->env, code);
} }
tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); pAPI->storeReader.storeReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); blockDistInfo.numOfInmemRows = (int32_t) pAPI->metaFn.getNumOfRowsInMem(pBlockScanInfo->pHandle);
SSDataBlock* pBlock = pBlockScanInfo->pResBlock; SSDataBlock* pBlock = pBlockScanInfo->pResBlock;
@ -2213,7 +2230,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
static void destroyBlockDistScanOperatorInfo(void* param) { static void destroyBlockDistScanOperatorInfo(void* param) {
SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param;
blockDataDestroy(pDistInfo->pResBlock); blockDataDestroy(pDistInfo->pResBlock);
tsdbReaderClose(pDistInfo->pHandle); pDistInfo->readHandle.api.storeReader.storeReaderClose(pDistInfo->pHandle);
tableListDestroy(pDistInfo->pTableListInfo); tableListDestroy(pDistInfo->pTableListInfo);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
@ -2268,7 +2285,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
size_t num = tableListGetSize(pTableListInfo); size_t num = tableListGetSize(pTableListInfo);
void* pList = tableListGetInfo(pTableListInfo, 0); void* pList = tableListGetInfo(pTableListInfo, 0);
code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false, NULL); code = readHandle->api.storeReader.storeReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL);
cleanupQueryTableDataCond(&cond); cleanupQueryTableDataCond(&cond);
if (code != 0) { if (code != 0) {
goto _error; goto _error;

1
source/libs/executor/src/timesliceoperator.c
View File

@ -18,6 +18,7 @@
#include "functionMgt.h" #include "functionMgt.h"
#include "operator.h" #include "operator.h"
#include "querytask.h" #include "querytask.h"
#include "storageapi.h"
#include "tcommon.h" #include "tcommon.h"
#include "tcompare.h" #include "tcompare.h"
#include "tdatablock.h" #include "tdatablock.h"

154
source/libs/executor/src/timewindowoperator.c
View File

@ -1331,10 +1331,12 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf,
} }
static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) {
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamIntervalOperatorInfo* pInfo = pOperator->info; SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SWinKey key = {.ts = ts, .groupId = groupId}; SWinKey key = {.ts = ts, .groupId = groupId};
tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));
streamStateDel(pInfo->pState, &key); pAPI->stateStore.streamStateDel(pInfo->pState, &key);
return true; return true;
} }
@ -1471,7 +1473,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
for (int32_t i = *index; i < size; i++) { for (int32_t i = *index; i < size; i++) {
SWinKey* pWin = taosArrayGet(pWins, i); SWinKey* pWin = taosArrayGet(pWins, i);
void* tbname = NULL; void* tbname = NULL;
streamStateGetParName(pInfo->pState, pWin->groupId, &tbname); pInfo->statestore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname);
if (tbname == NULL) { if (tbname == NULL) {
appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL); appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);
} else { } else {
@ -1479,7 +1481,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName); appendOneRowToStreamSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);
} }
streamFreeVal(tbname); pInfo->statestore.streamStateFreeVal(tbname);
(*index)++; (*index)++;
} }
} }
@ -1533,7 +1535,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
blockDataDestroy(pInfo->pPullDataRes); blockDataDestroy(pInfo->pPullDataRes);
taosArrayDestroy(pInfo->pDelWins); taosArrayDestroy(pInfo->pDelWins);
blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pDelRes);
streamFileStateDestroy(pInfo->pState->pFileState); pInfo->statestore.streamFileStateDestroy(pInfo->pState->pFileState);
taosMemoryFreeClear(pInfo->pState); taosMemoryFreeClear(pInfo->pState);
nodesDestroyNode((SNode*)pInfo->pPhyNode); nodesDestroyNode((SNode*)pInfo->pPhyNode);
@ -1608,16 +1610,20 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
} }
void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo) { void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo) {
SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
initIntervalDownStream(downstream->pDownstream[0], type, pInfo); initIntervalDownStream(downstream->pDownstream[0], type, pInfo);
return; return;
} }
SStreamScanInfo* pScanInfo = downstream->info; SStreamScanInfo* pScanInfo = downstream->info;
pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.parentType = type;
pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;
if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark); pScanInfo->pUpdateInfo = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark);
} }
pScanInfo->interval = pInfo->interval; pScanInfo->interval = pInfo->interval;
pScanInfo->twAggSup = pInfo->twAggSup; pScanInfo->twAggSup = pInfo->twAggSup;
pScanInfo->pState = pInfo->pState; pScanInfo->pState = pInfo->pState;
@ -2045,33 +2051,33 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3
} }
} }
bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return streamStateCheck(pState, pKey); } bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { return pStore->streamStateCheck(pState, pKey); }
int32_t setIntervalOutputBuf(SStreamState* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId, int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,
SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset,
SAggSupporter* pAggSup) { SAggSupporter* pAggSup, SStateStore* pStore) {
SWinKey key = {
.ts = win->skey, SWinKey key = { .ts = win->skey, .groupId = groupId };
.groupId = groupId,
};
char* value = NULL; char* value = NULL;
int32_t size = pAggSup->resultRowSize; int32_t size = pAggSup->resultRowSize;
if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { if (pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
*pResult = (SRowBuffPos*)value; *pResult = (SRowBuffPos*)value;
SResultRow* res = (SResultRow*)((*pResult)->pRowBuff); SResultRow* res = (SResultRow*)((*pResult)->pRowBuff);
// set time window for current result // set time window for current result
res->win = (*win); res->win = (*win);
setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) { bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore) {
if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {
SWinKey key = {.ts = pWin->skey, .groupId = groupId}; SWinKey key = {.ts = pWin->skey, .groupId = groupId};
if (!hasIntervalWindow(pState, &key)) { if (!hasIntervalWindow(pState, &key, pStore)) {
return true; return true;
} }
return false; return false;
@ -2102,7 +2108,7 @@ static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {
clearDiskbasedBuf(pInfo->aggSup.pResultBuf); clearDiskbasedBuf(pInfo->aggSup.pResultBuf);
initResultRowInfo(&pInfo->binfo.resultRowInfo); initResultRowInfo(&pInfo->binfo.resultRowInfo);
pInfo->aggSup.currentPageId = -1; pInfo->aggSup.currentPageId = -1;
streamStateClear(pInfo->pState); pInfo->statestore.streamStateClear(pInfo->pState);
} }
static void clearSpecialDataBlock(SSDataBlock* pBlock) { static void clearSpecialDataBlock(SSDataBlock* pBlock) {
@ -2198,13 +2204,15 @@ static void clearFunctionContext(SExprSupp* pSup) {
} }
} }
int32_t getOutputBuf(SStreamState* pState, SRowBuffPos* pPos, SResultRow** pResult) { int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) {
return streamStateGetByPos(pState, pPos, (void**)pResult); return pStore->streamStateGetByPos(pState, pPos, (void**)pResult);
} }
int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo) { SGroupResInfo* pGroupResInfo) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SExprInfo* pExprInfo = pSup->pExprInfo; SExprInfo* pExprInfo = pSup->pExprInfo;
int32_t numOfExprs = pSup->numOfExprs; int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
@ -2215,7 +2223,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {
SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i); SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i);
SResultRow* pRow = NULL; SResultRow* pRow = NULL;
int32_t code = getOutputBuf(pState, pPos, &pRow); int32_t code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore);
uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId; uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId;
ASSERT(code == 0); ASSERT(code == 0);
doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
@ -2227,12 +2235,12 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
if (pBlock->info.id.groupId == 0) { if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = groupId; pBlock->info.id.groupId = groupId;
void* tbname = NULL; void* tbname = NULL;
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else { } else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
} }
streamFreeVal(tbname); pAPI->stateStore.streamStateFreeVal(tbname);
} else { } else {
// current value belongs to different group, it can't be packed into one datablock // current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.id.groupId != groupId) { if (pBlock->info.id.groupId != groupId) {
@ -2279,7 +2287,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void doBuildStreamIntervalResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,
SGroupResInfo* pGroupResInfo) { SGroupResInfo* pGroupResInfo) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// set output datablock version // set output datablock version
@ -2354,7 +2362,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
.groupId = groupId, .groupId = groupId,
}; };
void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && isClosed && !chIds) { if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->statestore) && isClosed && !chIds) {
SPullWindowInfo pull = { SPullWindowInfo pull = {
.window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
// add pull data request // add pull data request
@ -2385,7 +2393,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
} }
int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,
pSup->rowEntryInfoOffset, &pInfo->aggSup); pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->statestore);
pResult = (SResultRow*)pResPos->pRowBuff; pResult = (SResultRow*)pResPos->pRowBuff;
if (code != TSDB_CODE_SUCCESS || pResult == NULL) { if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
@ -2467,6 +2475,8 @@ static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) {
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info; SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SOperatorInfo* downstream = pOperator->pDownstream[0]; SOperatorInfo* downstream = pOperator->pDownstream[0];
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
@ -2505,8 +2515,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
} else { } else {
if (pInfo->twAggSup.maxTs > 0 && if (pInfo->twAggSup.maxTs > 0 &&
pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) {
streamStateCommit(pInfo->pState); pAPI->stateStore.streamStateCommit(pInfo->pState);
streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs; pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs;
} }
qDebug("===stream===interval final close"); qDebug("===stream===interval final close");
@ -2707,6 +2717,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
} }
pOperator->pTaskInfo = pTaskInfo; pOperator->pTaskInfo = pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval, pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,
.sliding = pIntervalPhyNode->sliding, .sliding = pIntervalPhyNode->sliding,
.intervalUnit = pIntervalPhyNode->intervalUnit, .intervalUnit = pIntervalPhyNode->intervalUnit,
@ -2743,9 +2755,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock); initBasicInfo(&pInfo->binfo, pResBlock);
pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); pInfo->pState = taosMemoryCalloc(1, sizeof(void));
*(pInfo->pState) = *(pTaskInfo->streamInfo.pState); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
streamStateSetNumber(pInfo->pState, -1); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pInfo->pState); pInfo->pState);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -2753,10 +2765,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
} }
initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs);
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
initResultRowInfo(&pInfo->binfo.resultRowInfo); initResultRowInfo(&pInfo->binfo.resultRowInfo);
pInfo->numOfChild = numOfChild; pInfo->numOfChild = numOfChild;
pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode); pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode);
@ -2773,6 +2784,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
if (!IS_FINAL_OP(pInfo) || numOfChild == 0) { if (!IS_FINAL_OP(pInfo) || numOfChild == 0) {
pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
} }
pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo)); pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));
pInfo->pullIndex = 0; pInfo->pullIndex = 0;
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
@ -2789,7 +2801,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pInfo->pUpdated = NULL; pInfo->pUpdated = NULL;
pInfo->pUpdatedMap = NULL; pInfo->pUpdatedMap = NULL;
int32_t funResSize= getMaxFunResSize(&pOperator->exprSupp, numOfCols); int32_t funResSize= getMaxFunResSize(&pOperator->exprSupp, numOfCols);
pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, pInfo->pState->pFileState = pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); compareTs, pInfo->pState, pInfo->twAggSup.deleteMark);
pInfo->dataVersion = 0; pInfo->dataVersion = 0;
@ -2887,7 +2899,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin
pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
pScanInfo->pState = pAggSup->pState; pScanInfo->pState = pAggSup->pState;
if ((!pScanInfo->igCheckUpdate || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) && !pScanInfo->pUpdateInfo) { if ((!pScanInfo->igCheckUpdate || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) && !pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark);
} }
pScanInfo->twAggSup = *pTwSup; pScanInfo->twAggSup = *pTwSup;
} }
@ -2905,9 +2917,9 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx,
} }
initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput); initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput);
pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); pSup->pState = taosMemoryCalloc(1, sizeof(void));
*(pSup->pState) = *pState; *(pSup->pState) = *pState;
streamStateSetNumber(pSup->pState, -1); pSup->stateStore.streamStateSetNumber(pSup->pState, -1);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pSup->pResultRows = tSimpleHashInit(32, hashFn); pSup->pResultRows = tSimpleHashInit(32, hashFn);
@ -2950,7 +2962,7 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT
pKey->win.skey = startTs; pKey->win.skey = startTs;
pKey->win.ekey = endTs; pKey->win.ekey = endTs;
pKey->groupId = groupId; pKey->groupId = groupId;
int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey); int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pKey); SET_SESSION_WIN_KEY_INVALID(pKey);
} }
@ -2964,8 +2976,8 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT
pCurWin->sessionWin.win.skey = startTs; pCurWin->sessionWin.win.skey = startTs;
pCurWin->sessionWin.win.ekey = endTs; pCurWin->sessionWin.win.ekey = endTs;
int32_t size = pAggSup->resultRowSize; int32_t size = pAggSup->resultRowSize;
int32_t code = int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin,
streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size); pAggSup->gap, &pCurWin->pOutputBuf, &size);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pCurWin->isOutput = true; pCurWin->isOutput = true;
} else { } else {
@ -2976,11 +2988,12 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT
int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) {
int32_t size = 0; int32_t size = 0;
int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size); int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
streamStateCurNext(pAggSup->pState, pCur);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void saveDeleteInfo(SArray* pWins, SSessionKey key) { void saveDeleteInfo(SArray* pWins, SSessionKey key) {
@ -3065,7 +3078,7 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo
} }
static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {
streamStateSessionDel(pAggSup->pState, pKey); pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey);
SSessionKey hashKey = {0}; SSessionKey hashKey = {0};
getSessionHashKey(pKey, &hashKey); getSessionHashKey(pKey, &hashKey);
tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey));
@ -3083,12 +3096,12 @@ static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo*
SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,
SResultWindowInfo* pNextWin) { SResultWindowInfo* pNextWin) {
SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);
pNextWin->isOutput = true; pNextWin->isOutput = true;
setSessionWinOutputInfo(pStUpdated, pNextWin); setSessionWinOutputInfo(pStUpdated, pNextWin);
int32_t size = 0; int32_t size = 0;
pNextWin->sessionWin = pCurWin->sessionWin; pNextWin->sessionWin = pCurWin->sessionWin;
int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size); int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pNextWin->pOutputBuf); taosMemoryFreeClear(pNextWin->pOutputBuf);
SET_SESSION_WIN_INVALID(*pNextWin); SET_SESSION_WIN_INVALID(*pNextWin);
@ -3100,6 +3113,8 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC
SSHashObj* pStDeleted) { SSHashObj* pStDeleted) {
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SResultRow* pCurResult = NULL; SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
@ -3111,7 +3126,7 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC
SStreamStateCur* pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo); SStreamStateCur* pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);
if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) { if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) {
taosMemoryFree(winInfo.pOutputBuf); taosMemoryFree(winInfo.pOutputBuf);
streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
break; break;
} }
SResultRow* pWinResult = NULL; SResultRow* pWinResult = NULL;
@ -3125,13 +3140,13 @@ static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pC
} }
removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin); removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin);
doDeleteSessionWindow(pAggSup, &winInfo.sessionWin); doDeleteSessionWindow(pAggSup, &winInfo.sessionWin);
streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
taosMemoryFree(winInfo.pOutputBuf); taosMemoryFree(winInfo.pOutputBuf);
} }
} }
int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) { int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) {
saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize); saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize, &pAggSup->stateStore);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -3251,6 +3266,8 @@ static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) {
} }
void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) {
SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;
blockDataCleanup(pBlock); blockDataCleanup(pBlock);
int32_t size = tSimpleHashGetSize(pStDeleted); int32_t size = tSimpleHashGetSize(pStDeleted);
if (size == 0) { if (size == 0) {
@ -3279,14 +3296,14 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo
SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
void* tbname = NULL; void* tbname = NULL;
streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname); pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname);
if (tbname == NULL) { if (tbname == NULL) {
colDataSetNULL(pTableCol, pBlock->info.rows); colDataSetNULL(pTableCol, pBlock->info.rows);
} else { } else {
char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
streamFreeVal(tbname); pAPI->stateStore.streamStateFreeVal(tbname);
} }
pBlock->info.rows += 1; pBlock->info.rows += 1;
} }
@ -3298,6 +3315,8 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo
static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) {
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
int32_t size = taosArrayGetSize(pWinArray); int32_t size = taosArrayGetSize(pWinArray);
SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
@ -3314,7 +3333,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS
SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup; SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup;
SSessionKey chWinKey = {0}; SSessionKey chWinKey = {0};
getSessionHashKey(pWinKey, &chWinKey); getSessionHashKey(pWinKey, &chWinKey);
SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);
SResultRow* pResult = NULL; SResultRow* pResult = NULL;
SResultRow* pChResult = NULL; SResultRow* pChResult = NULL;
while (1) { while (1) {
@ -3341,7 +3360,7 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS
break; break;
} }
} }
streamStateFreeCur(pCur); pAPI->stateStore.streamStateFreeCur(pCur);
} }
if (num > 0) { if (num > 0) {
saveSessionOutputBuf(pAggSup, &parentWin); saveSessionOutputBuf(pAggSup, &parentWin);
@ -3406,7 +3425,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL
pGroupResInfo->pBuf = NULL; pGroupResInfo->pBuf = NULL;
} }
void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo, void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo,
SSDataBlock* pBlock) { SSDataBlock* pBlock) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// set output datablock version // set output datablock version
@ -3636,7 +3655,7 @@ _error:
static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {
tSimpleHashClear(pInfo->streamAggSup.pResultRows); tSimpleHashClear(pInfo->streamAggSup.pResultRows);
streamStateSessionClear(pInfo->streamAggSup.pState); pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState);
} }
static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
@ -3762,6 +3781,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
goto _error; goto _error;
} }
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION); pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION);
@ -3786,7 +3806,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream
} }
SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;
streamStateSetNumber(pChInfo->streamAggSup.pState, i); pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i);
taosArrayPush(pInfo->pChildren, &pChildOp); taosArrayPush(pInfo->pChildren, &pChildOp);
} }
} }
@ -3849,7 +3869,7 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.skey = ts;
pCurWin->winInfo.sessionWin.win.ekey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts;
int32_t code = int32_t code =
streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize, pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize,
compareStateKey, &pCurWin->winInfo.pOutputBuf, &size); compareStateKey, &pCurWin->winInfo.pOutputBuf, &size);
pCurWin->pStateKey = pCurWin->pStateKey =
(SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize));
@ -3870,12 +3890,12 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;
pNextWin->winInfo.pOutputBuf = NULL; pNextWin->winInfo.pOutputBuf = NULL;
SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_INVALID(pNextWin->winInfo); SET_SESSION_WIN_INVALID(pNextWin->winInfo);
} }
streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
} }
int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId, int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId,
@ -3914,7 +3934,9 @@ int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNex
static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,
SSHashObj* pStDeleted) { SSHashObj* pStDeleted) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SStreamStateAggOperatorInfo* pInfo = pOperator->info; SStreamStateAggOperatorInfo* pInfo = pOperator->info;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
uint64_t groupId = pSDataBlock->info.id.groupId; uint64_t groupId = pSDataBlock->info.id.groupId;
@ -3956,7 +3978,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);
releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf); releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf, &pAPI->stateStore);
continue; continue;
} }
code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
@ -4736,6 +4758,8 @@ _error:
static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info; SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
@ -4757,8 +4781,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
if (pInfo->twAggSup.maxTs > 0 && if (pInfo->twAggSup.maxTs > 0 &&
pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) {
streamStateCommit(pInfo->pState); pAPI->stateStore.streamStateCommit(pInfo->pState);
streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark); pAPI->stateStore.streamStateDeleteCheckPoint(pInfo->pState, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark);
setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId);
pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs; pInfo->twAggSup.checkPointTs = pInfo->twAggSup.maxTs;
} }
@ -4900,6 +4924,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
pOperator->pTaskInfo = pTaskInfo; pOperator->pTaskInfo = pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
pInfo->ignoreExpiredDataSaved = false; pInfo->ignoreExpiredDataSaved = false;
pInfo->isFinal = false; pInfo->isFinal = false;
@ -4912,9 +4938,9 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); pInfo->pState = taosMemoryCalloc(1, sizeof(void));
*(pInfo->pState) = *(pTaskInfo->streamInfo.pState); *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);
streamStateSetNumber(pInfo->pState, -1); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
@ -4952,7 +4978,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->pUpdated = NULL; pInfo->pUpdated = NULL;
pInfo->pUpdatedMap = NULL; pInfo->pUpdatedMap = NULL;
int32_t funResSize= getMaxFunResSize(pSup, numOfCols); int32_t funResSize= getMaxFunResSize(pSup, numOfCols);
pInfo->pState->pFileState = streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, pInfo->pState->pFileState = pTaskInfo->storageAPI.stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,
compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); compareTs, pInfo->pState, pInfo->twAggSup.deleteMark);
setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,

1
source/libs/function/CMakeLists.txt
View File

@ -33,7 +33,6 @@ target_link_libraries(
PRIVATE qcom PRIVATE qcom
PRIVATE scalar PRIVATE scalar
PRIVATE transport PRIVATE transport
PRIVATE stream ${LINK_JEMALLOC}
PUBLIC uv_a PUBLIC uv_a
) )

12
source/libs/function/src/builtinsimpl.c
View File

@ -18,7 +18,7 @@
#include "function.h" #include "function.h"
#include "query.h" #include "query.h"
#include "querynodes.h" #include "querynodes.h"
#include "streamState.h" //#include "streamState.h"
#include "tcompare.h" #include "tcompare.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tdigest.h" #include "tdigest.h"
@ -3155,9 +3155,9 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf,
releaseBufPage(pHandle->pBuf, pPage); releaseBufPage(pHandle->pBuf, pPage);
} else { } else {
// other tuple save policy // other tuple save policy
if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { // if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) {
p.streamTupleKey = *key; // p.streamTupleKey = *key;
} // }
} }
*pPos = p; *pPos = p;
@ -3192,7 +3192,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf
setBufPageDirty(pPage, true); setBufPageDirty(pPage, true);
releaseBufPage(pHandle->pBuf, pPage); releaseBufPage(pHandle->pBuf, pPage);
} else { } else {
streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); // streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -3217,7 +3217,7 @@ static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPo
} else { } else {
void* value = NULL; void* value = NULL;
int32_t vLen; int32_t vLen;
streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); // streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen);
return (char*)value; return (char*)value;
} }
} }

2
source/libs/index/CMakeLists.txt
View File

@ -12,7 +12,7 @@ target_link_libraries(
PUBLIC os PUBLIC os
PUBLIC util PUBLIC util
PUBLIC common PUBLIC common
PUBLIC vnode # PUBLIC vnode
PUBLIC nodes PUBLIC nodes
PUBLIC scalar PUBLIC scalar
PUBLIC function PUBLIC function

5
source/libs/index/src/indexFilter.c
View File

@ -20,7 +20,7 @@
#include "querynodes.h" #include "querynodes.h"
#include "scalar.h" #include "scalar.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "vnode.h" #include "tsdstorage.h"
// clang-format off // clang-format off
#define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
@ -79,6 +79,7 @@ typedef struct SIFParam {
char colName[TSDB_COL_NAME_LEN * 2 + 4]; char colName[TSDB_COL_NAME_LEN * 2 + 4];
SIndexMetaArg arg; SIndexMetaArg arg;
SStoreAPI api;
} SIFParam; } SIFParam;
typedef struct SIFCtx { typedef struct SIFCtx {
@ -659,7 +660,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
} else { } else {
if (sifSetFltParam(left, right, &typedata, &param) != 0) return -1; if (sifSetFltParam(left, right, &typedata, &param) != 0) return -1;
} }
ret = metaFilterTableIds(arg->metaEx, &param, output->result); ret = left->api.metaFilterTableIds(arg->metaEx, &param, output->result);
} }
return ret; return ret;
} }

1
source/libs/scalar/CMakeLists.txt
View File

@ -14,6 +14,7 @@ target_link_libraries(scalar
PRIVATE nodes PRIVATE nodes
PRIVATE function PRIVATE function
PRIVATE qcom PRIVATE qcom
PRIVATE parser
) )
if(${BUILD_TEST}) if(${BUILD_TEST})

1
source/libs/stream/inc/streamBackendRocksdb.h
View File

@ -17,7 +17,6 @@
#define _STREAM_BACKEDN_ROCKSDB_H_ #define _STREAM_BACKEDN_ROCKSDB_H_
#include "rocksdb/c.h" #include "rocksdb/c.h"
// #include "streamInc.h"
#include "streamState.h" #include "streamState.h"
#include "tcoding.h" #include "tcoding.h"
#include "tcommon.h" #include "tcommon.h"

18
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -214,8 +214,7 @@ int streamGetInit(const char* funcName);
// |key|-----value------| // |key|-----value------|
// |key|ttl|len|userData| // |key|ttl|len|userData|
static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, void** snapshot, void** readOpt);
rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt);
int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) { int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf, size_t bLen) {
int ret = memcmp(aBuf, bBuf, aLen); int ret = memcmp(aBuf, bBuf, aLen);
@ -799,10 +798,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
if (ppInst != NULL && *ppInst != NULL) { if (ppInst != NULL && *ppInst != NULL) {
RocksdbCfInst* inst = *ppInst; RocksdbCfInst* inst = *ppInst;
pState->pTdbState->rocksdb = inst->db; pState->pTdbState->rocksdb = inst->db;
pState->pTdbState->pHandle = inst->pHandle; pState->pTdbState->pHandle = (void**)inst->pHandle;
pState->pTdbState->writeOpts = inst->wOpt; pState->pTdbState->writeOpts = inst->wOpt;
pState->pTdbState->readOpts = inst->rOpt; pState->pTdbState->readOpts = inst->rOpt;
pState->pTdbState->cfOpts = inst->cfOpt; pState->pTdbState->cfOpts = (void**)inst->cfOpt;
pState->pTdbState->dbOpt = handle->dbOpt; pState->pTdbState->dbOpt = handle->dbOpt;
pState->pTdbState->param = inst->param; pState->pTdbState->param = inst->param;
pState->pTdbState->pBackendHandle = handle; pState->pTdbState->pBackendHandle = handle;
@ -810,6 +809,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
taosThreadMutexUnlock(&handle->cfMutex); taosThreadMutexUnlock(&handle->cfMutex);
return 0; return 0;
} }
taosThreadMutexUnlock(&handle->cfMutex); taosThreadMutexUnlock(&handle->cfMutex);
char* err = NULL; char* err = NULL;
@ -850,10 +850,10 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
} }
} }
pState->pTdbState->rocksdb = handle->db; pState->pTdbState->rocksdb = handle->db;
pState->pTdbState->pHandle = cfHandle; pState->pTdbState->pHandle = (void**)cfHandle;
pState->pTdbState->writeOpts = rocksdb_writeoptions_create(); pState->pTdbState->writeOpts = rocksdb_writeoptions_create();
pState->pTdbState->readOpts = rocksdb_readoptions_create(); pState->pTdbState->readOpts = rocksdb_readoptions_create();
pState->pTdbState->cfOpts = (rocksdb_options_t**)cfOpt; pState->pTdbState->cfOpts = (void**)(rocksdb_options_t**)cfOpt;
pState->pTdbState->dbOpt = handle->dbOpt; pState->pTdbState->dbOpt = handle->dbOpt;
pState->pTdbState->param = param; pState->pTdbState->param = param;
pState->pTdbState->pBackendHandle = handle; pState->pTdbState->pBackendHandle = handle;
@ -954,8 +954,7 @@ bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len
} }
return true; return true;
} }
rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, void** snapshot, void** readOpt) {
rocksdb_readoptions_t** readOpt) {
int idx = streamGetInit(cfName); int idx = streamGetInit(cfName);
if (snapshot != NULL) { if (snapshot != NULL) {
@ -1896,7 +1895,7 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co
rocksdb_snapshot_t* snapshot = NULL; rocksdb_snapshot_t* snapshot = NULL;
rocksdb_readoptions_t* readopts = NULL; rocksdb_readoptions_t* readopts = NULL;
rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", &snapshot, &readopts); rocksdb_iterator_t* pIter = streamStateIterCreate(pState, "default", (void**)&snapshot, (void**)&readopts);
if (pIter == NULL) { if (pIter == NULL) {
return -1; return -1;
} }
@ -1966,6 +1965,7 @@ char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len) {
} }
return dst; return dst;
} }
// batch func // batch func
void* streamStateCreateBatch() { void* streamStateCreateBatch() {
rocksdb_writebatch_t* pBatch = rocksdb_writebatch_create(); rocksdb_writebatch_t* pBatch = rocksdb_writebatch_create();

1
source/libs/stream/src/tstreamFileState.c
View File

@ -21,6 +21,7 @@
#include "tcommon.h" #include "tcommon.h"
#include "thash.h" #include "thash.h"
#include "tsimplehash.h" #include "tsimplehash.h"
#include "storageapi.h"
#define FLUSH_RATIO 0.5 #define FLUSH_RATIO 0.5
#define FLUSH_NUM 4 #define FLUSH_NUM 4