xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli

This commit is contained in:
Minghao Li 2022-03-05 15:06:59 +08:00
parent b7e22154b2 9f33b3ea1e
commit 29fab2e30a
150 changed files with 4083 additions and 2628 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/test/craft/CMakeLists.txt
View File

@ -1,2 +1,2 @@
add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode craft lz4 uv_a)
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)

2
contrib/test/traft/single_node/CMakeLists.txt
View File

@ -3,4 +3,4 @@ target_sources(singleNode
PRIVATE
"singleNode.c"
)
target_link_libraries(singleNode traft lz4 uv_a)
target_link_libraries(singleNode PUBLIC traft lz4 uv_a)

23
include/common/tcommon.h
View File

@ -25,24 +25,6 @@
extern "C" {
#endif
// typedef struct STimeWindow {
// TSKEY skey;
// TSKEY ekey;
// } STimeWindow;
// typedef struct {
// int32_t dataLen;
// char name[TSDB_TABLE_FNAME_LEN];
// char *data;
// } STagData;
// typedef struct SSchema {
// uint8_t type;
// char name[TSDB_COL_NAME_LEN];
// int16_t colId;
// int16_t bytes;
// } SSchema;
enum {
TMQ_CONF__RESET_OFFSET__LATEST = -1,
TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
@ -50,7 +32,8 @@ enum {
};
enum {
TMQ_MSG_TYPE__POLL_RSP = 0,
TMQ_MSG_TYPE__DUMMY = 0,
TMQ_MSG_TYPE__POLL_RSP,
TMQ_MSG_TYPE__EP_RSP,
};
@ -285,4 +268,4 @@ typedef struct SSessionWindow {
}
#endif
#endif /*_TD_COMMON_DEF_H_*/
#endif /*_TD_COMMON_DEF_H_*/

0
include/common/tep.h → include/common/tdatablock.h
View File

2
include/common/tmsg.h
View File

@ -70,7 +70,7 @@ typedef uint16_t tmsg_t;
typedef enum {
HEARTBEAT_TYPE_MQ = 0,
HEARTBEAT_TYPE_QUERY = 1,
HEARTBEAT_TYPE_QUERY,
// types can be added here
//
HEARTBEAT_TYPE_MAX

19
include/libs/executor/executor.h
View File

@ -89,25 +89,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds);
*/
int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext);
/**
*
* Retrieve the actual results to fill the response message payload.
* Note that this function must be executed after qRetrieveQueryResultInfo is invoked.
*
* @param tinfo tinfo object
* @param pRsp response message
* @param contLen payload length
* @return
*/
//int32_t qDumpRetrieveResult(qTaskInfo_t tinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
/**
* return the transporter context (RPC)
* @param tinfo
* @return
*/
void* qGetResultRetrieveMsg(qTaskInfo_t tinfo);
/**
* kill the ongoing query and free the query handle and corresponding resources automatically
* @param tinfo qhandle

39
include/libs/monitor/monitor.h
View File

@ -18,6 +18,7 @@
#include "tarray.h"
#include "tdef.h"
#include "tlog.h"
#ifdef __cplusplus
extern "C" {
@ -85,21 +86,21 @@ typedef struct {
typedef struct {
float uptime; // day
float cpu_engine;
float cpu_system;
double cpu_engine;
double cpu_system;
float cpu_cores;
float mem_engine; // MB
float mem_system; // MB
float mem_total; // MB
float disk_engine; // GB
float disk_used; // GB
float disk_total; // GB
float net_in; // Kb/s
float net_out; // Kb/s
float io_read; // Mb/s
float io_write; // Mb/s
float io_read_disk; // Mb/s
float io_write_disk; // Mb/s
int64_t mem_engine; // KB
int64_t mem_system; // KB
int64_t mem_total; // KB
int64_t disk_engine; // Byte
int64_t disk_used; // Byte
int64_t disk_total; // Byte
double net_in; // bytes per second
double net_out; // bytes per second
double io_read;
double io_write;
double io_read_disk;
double io_write_disk;
int32_t req_select;
float req_select_rate;
int32_t req_insert;
@ -116,7 +117,7 @@ typedef struct {
typedef struct {
char name[TSDB_FILENAME_LEN];
int32_t level;
int8_t level;
SDiskSize size;
} SMonDiskDesc;
@ -126,12 +127,6 @@ typedef struct {
SMonDiskDesc tempdir;
} SMonDiskInfo;
typedef struct {
int64_t ts;
int8_t level;
char content[MON_LOG_LEN];
} SMonLogItem;
typedef struct SMonInfo SMonInfo;
typedef struct {
@ -142,7 +137,7 @@ typedef struct {
int32_t monInit(const SMonCfg *pCfg);
void monCleanup();
void monAddLogItem(SMonLogItem *pItem);
void monRecordLog(int64_t ts, ELogLevel level, const char *content);
SMonInfo *monCreateMonitorInfo();
void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo);

16
include/libs/qcom/query.h
View File

@ -168,20 +168,18 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl
extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize);
#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE
#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE
#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
#define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); }} while(0)
#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); }} while(0)
#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); }} while(0)
#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus
}

5
include/libs/scalar/filter.h
View File

@ -19,10 +19,12 @@
extern "C" {
#endif
#include "tcommon.h"
#include "nodes.h"
typedef struct SFilterInfo SFilterInfo;
typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
enum {
FLT_OPTION_NO_REWRITE = 1,
FLT_OPTION_TIMESTAMP = 2,
@ -34,7 +36,6 @@ typedef struct SFilterColumnParam{
SArray* pDataBlock;
} SFilterColumnParam;
extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param);

2
include/libs/sync/sync.h
View File

@ -21,7 +21,7 @@ extern "C" {
#endif
#include <stdint.h>
#include <tep.h>
#include <tdatablock.h>
#include "taosdef.h"
#include "trpc.h"

9
include/libs/tfs/tfs.h
View File

@ -17,6 +17,7 @@
#define _TD_TFS_H_
#include "tdef.h"
#include "monitor.h"
#ifdef __cplusplus
extern "C" {
@ -237,6 +238,14 @@ const STfsFile *tfsReaddir(STfsDir *pDir);
*/
void tfsClosedir(STfsDir *pDir);
/**
* @brief Get disk info of tfs.
*
* @param pTfs The fs object.
* @param pInfo The info object.
*/
int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo);
#ifdef __cplusplus
}
#endif

62
include/libs/wal/wal.h
View File

@ -24,43 +24,41 @@
extern "C" {
#endif
extern int32_t wDebugFlag;
#define wFatal(...) \
{ \
if (wDebugFlag & DEBUG_FATAL) { \
taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \
} \
#define wFatal(...) \
{ \
if (wDebugFlag & DEBUG_FATAL) { \
taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
} \
}
#define wError(...) \
{ \
if (wDebugFlag & DEBUG_ERROR) { \
taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \
} \
#define wError(...) \
{ \
if (wDebugFlag & DEBUG_ERROR) { \
taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \
}
#define wWarn(...) \
{ \
if (wDebugFlag & DEBUG_WARN) { \
taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \
} \
#define wWarn(...) \
{ \
if (wDebugFlag & DEBUG_WARN) { \
taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \
}
#define wInfo(...) \
{ \
if (wDebugFlag & DEBUG_INFO) { \
taosPrintLog("WAL ", 255, __VA_ARGS__); \
} \
#define wInfo(...) \
{ \
if (wDebugFlag & DEBUG_INFO) { \
taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \
} \
}
#define wDebug(...) \
{ \
if (wDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
} \
#define wDebug(...) \
{ \
if (wDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \
} \
}
#define wTrace(...) \
{ \
if (wDebugFlag & DEBUG_TRACE) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
} \
#define wTrace(...) \
{ \
if (wDebugFlag & DEBUG_TRACE) { \
taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \
} \
}
#define WAL_HEAD_VER 0

1
include/os/os.h
View File

@ -44,6 +44,7 @@ extern "C" {
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <sys/param.h>
#include <unistd.h>
#include <wchar.h>
#include <wctype.h>

2
include/os/osDir.h
View File

@ -21,7 +21,7 @@ extern "C" {
#endif
void taosRemoveDir(const char *dirname);
int32_t taosDirExist(char *dirname);
bool taosDirExist(char *dirname);
int32_t taosMkDir(const char *dirname);
void taosRemoveOldFiles(const char *dirname, int32_t keepDays);
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);

6
include/os/osEnv.h
View File

@ -28,11 +28,11 @@ extern char tsCharset[];
extern char tsLocale[];
extern int8_t tsDaylight;
extern bool tsEnableCoreFile;
extern int64_t tsPageSize;
extern int64_t tsPageSizeKB;
extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern int32_t tsNumOfCores;
extern int32_t tsTotalMemoryMB;
extern float tsNumOfCores;
extern int64_t tsTotalMemoryKB;
extern char configDir[];
extern char tsDataDir[];

30
include/os/osFile.h
View File

@ -22,11 +22,18 @@ extern "C" {
#include "osSocket.h"
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID
#define fopen FOPEN_FUNC_TAOS_FORBID
// #define close CLOSE_FUNC_TAOS_FORBID
// #define fclose FCLOSE_FUNC_TAOS_FORBID
#define access ACCESS_FUNC_TAOS_FORBID
#define stat STAT_FUNC_TAOS_FORBID
#define lstat LSTAT_FUNC_TAOS_FORBID
#define fstat FSTAT_FUNC_TAOS_FORBID
#define close CLOSE_FUNC_TAOS_FORBID
#define fclose FCLOSE_FUNC_TAOS_FORBID
#define fsync FSYNC_FUNC_TAOS_FORBID
// #define fflush FFLUSH_FUNC_TAOS_FORBID
#endif
#ifndef PATH_MAX
@ -43,7 +50,13 @@ typedef struct TdFile *TdFilePtr;
#define TD_FILE_TEXT 0x0020
#define TD_FILE_AUTO_DEL 0x0040
#define TD_FILE_EXCL 0x0080
#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile
#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
#define TD_FILE_ACCESS_EXIST_OK 0x1
#define TD_FILE_ACCESS_READ_OK 0x2
#define TD_FILE_ACCESS_WRITE_OK 0x4
bool taosCheckAccessFile(const char *pathname, int mode);
int32_t taosLockFile(TdFilePtr pFile);
int32_t taosUnLockFile(TdFilePtr pFile);
@ -51,9 +64,9 @@ int32_t taosUnLockFile(TdFilePtr pFile);
int32_t taosUmaskFile(int32_t maskVal);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno);
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
bool taosCheckExistFile(const char *pathname);
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length);
@ -62,7 +75,7 @@ int32_t taosFsyncFile(TdFilePtr pFile);
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
int32_t taosEOFFile(TdFilePtr pFile);
@ -70,8 +83,9 @@ int64_t taosCloseFile(TdFilePtr *ppFile);
int32_t taosRenameFile(const char *oldName, const char *newName);
int64_t taosCopyFile(const char *from, const char *to);
int32_t taosRemoveFile(const char *path);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
@ -79,7 +93,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
bool taosValidFile(TdFilePtr pFile);
int taosGetErrorFile(TdFilePtr pFile);
int32_t taosGetErrorFile(TdFilePtr pFile);
#ifdef __cplusplus
}

24
include/os/osSysinfo.h
View File

@ -34,19 +34,19 @@ typedef struct {
} SDiskSpace;
void taosGetSystemInfo();
bool taosGetEmail(char *email, int32_t maxLen);
bool taosGetOsReleaseName(char *releaseName, int32_t maxLen);
bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores);
int32_t taosGetCpuCores();
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
bool taosGetTotalSysMemoryKB(uint64_t *kb);
bool taosGetProcMemory(float *memoryUsedMB); //
bool taosGetSysMemory(float *memoryUsedMB); //
int32_t taosGetEmail(char *email, int32_t maxLen);
int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
int32_t taosGetCpuCores(float *numOfCores);
int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine);
int32_t taosGetTotalMemory(int64_t *totalKB);
int32_t taosGetProcMemory(int64_t *usedKB);
int32_t taosGetSysMemory(int64_t *usedKB);
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
bool taosReadProcIO(int64_t *rchars, int64_t *wchars);
bool taosGetProcIO(float *readKB, float *writeKB);
bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
bool taosGetBandSpeed(float *bandSpeedKb);
int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
int32_t taosGetIOSpeed(double *readKB, double *writeKB, double *readDiskKB, double *writeDiskKB);
int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
int32_t taosGetBandSpeed(double *receive_bytes_per_sec, double *transmit_bytes_per_sec);
int32_t taosSystem(const char *cmd);
void taosKillSystem();

67
include/util/tcache.h
View File

@ -40,55 +40,9 @@ typedef struct SCacheStatis {
int64_t refreshCount;
} SCacheStatis;
struct STrashElem;
typedef struct SCacheDataNode {
uint64_t addedTime; // the added time when this element is added or updated into cache
uint64_t lifespan; // life duration when this element should be remove from cache
uint64_t expireTime; // expire time
uint64_t signature;
struct STrashElem *pTNodeHeader; // point to trash node head
uint16_t keySize : 15; // max key size: 32kb
bool inTrashcan : 1; // denote if it is in trash or not
uint32_t size; // allocated size for current SCacheDataNode
T_REF_DECLARE()
char *key;
char data[];
} SCacheDataNode;
typedef struct STrashElem {
struct STrashElem *prev;
struct STrashElem *next;
SCacheDataNode *pData;
} STrashElem;
/*
* to accommodate the old data which has the same key value of new one in hashList
* when an new node is put into cache, if an existed one with the same key:
* 1. if the old one does not be referenced, update it.
* 2. otherwise, move the old one to pTrash, addedTime the new one.
*
* when the node in pTrash does not be referenced, it will be release at the expired expiredTime
*/
typedef struct {
int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included.
int64_t refreshTime;
STrashElem *pTrash;
char *name;
SCacheStatis statistics;
SHashObj *pHashTable;
__cache_free_fn_t freeFp;
uint32_t numOfElemsInTrash; // number of element in trash
uint8_t deleting; // set the deleting flag to stop refreshing ASAP.
pthread_t refreshWorker;
bool extendLifespan; // auto extend life span when one item is accessed.
int64_t checkTick; // tick used to record the check times of the refresh threads
#if defined(LINUX)
pthread_rwlock_t lock;
#else
pthread_mutex_t lock;
#endif
} SCacheObj;
typedef struct SCacheObj SCacheObj;
typedef struct SCacheIter SCacheIter;
typedef struct STrashElem STrashElem;
/**
* initialize the cache object
@ -141,7 +95,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data);
* @param data
* @return
*/
void *taosCacheTransfer(SCacheObj *pCacheObj, void **data);
void *taosCacheTransferData(SCacheObj *pCacheObj, void **data);
/**
* remove data in cache, the data will not be removed immediately.
@ -152,6 +106,13 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data);
*/
void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
/**
*
* @param pCacheObj
* @return
*/
size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
/**
* move all data node into trash, clear node in trash can if it is not referenced by any clients
* @param handle
@ -184,6 +145,12 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
*/
void taosStopCacheRefreshWorker();
SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
bool taosCacheIterNext(SCacheIter* pIter);
void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
void taosCacheDestroyIter(SCacheIter* pIter);
#ifdef __cplusplus
}
#endif

105
include/util/thash.h
View File

@ -28,12 +28,8 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_before_fn_t)(void *);
typedef void (*_hash_free_fn_t)(void *);
#define HASH_MAX_CAPACITY (1024 * 1024 * 16)
#define HASH_DEFAULT_LOAD_FACTOR (0.75)
#define HASH_INDEX(v, c) ((v) & ((c)-1))
#define HASH_NODE_EXIST(code) (code == -2)
#define HASH_KEY_ALREADY_EXISTS (-2)
#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS)
/**
* murmur hash algorithm
@ -54,49 +50,15 @@ uint32_t taosIntHash_32(const char *key, uint32_t len);
uint32_t taosIntHash_64(const char *key, uint32_t len);
_hash_fn_t taosGetDefaultHashFunction(int32_t type);
_equal_fn_t taosGetDefaultEqualFunction(int32_t type);
typedef struct SHashNode {
struct SHashNode *next;
uint32_t hashVal; // the hash value of key
uint32_t dataLen; // length of data
uint32_t keyLen; // length of the key
uint16_t count; // reference count
int8_t removed; // flag to indicate removed
char data[];
} SHashNode;
#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen)
#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode)))
typedef enum SHashLockTypeE {
HASH_NO_LOCK = 0,
HASH_ENTRY_LOCK = 1,
} SHashLockTypeE;
typedef struct SHashEntry {
int32_t num; // number of elements in current entry
SRWLatch latch; // entry latch
SHashNode *next;
} SHashEntry;
typedef struct SHashObj {
SHashEntry **hashList;
uint32_t capacity; // number of slots
uint32_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function
_hash_free_fn_t freeFp; // hash node free callback function
_equal_fn_t equalFp; // equal function
_hash_before_fn_t callbackFp; // function invoked before return the value to caller
SRWLatch lock; // read-write spin lock
SHashLockTypeE type; // lock type
bool enableUpdate; // enable update
SArray *pMemBlock; // memory block allocated for SHashEntry
} SHashObj;
typedef struct SHashNode SHashNode;
typedef struct SHashObj SHashObj;
/**
* init the hash table
@ -126,8 +88,6 @@ int32_t taosHashGetSize(const SHashObj *pHashObj);
*/
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size);
int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded);
/**
* return the payload data with the specified key
*
@ -146,17 +106,18 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen);
* @param destBuf
* @return
*/
void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf);
int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf);
/**
* Clone the result to interval allocated buffer
*
* @param pHashObj
* @param key
* @param keyLen
* @param destBuf
* @param size
* @return
*/
void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, size_t *sz);
int32_t taosHashGetDup_m(SHashObj* pHashObj, const void* key, size_t keyLen, void** destBuf, int32_t* size);
/**
* remove item with the specified key
@ -207,37 +168,13 @@ void *taosHashIterate(SHashObj *pHashObj, void *p);
*/
void taosHashCancelIterate(SHashObj *pHashObj, void *p);
/**
* Get the corresponding key information for a given data in hash table
* @param data
* @return
*/
int32_t taosHashGetKey(void *data, void **key, size_t *keyLen);
/**
* Get the corresponding key information for a given data in hash table, using memcpy
* @param data
* @param dst
* @return
*/
static FORCE_INLINE int32_t taosHashCopyKey(void *data, void *dst) {
if (NULL == data || NULL == dst) {
return -1;
}
SHashNode *node = GET_HASH_PNODE(data);
void *key = GET_HASH_NODE_KEY(node);
memcpy(dst, key, node->keyLen);
return 0;
}
/**
* Get the corresponding data length for a given data in hash table
* @param data
* @return
*/
int32_t taosHashGetDataLen(void *data);
/**
* Get the corresponding key information for a given data in hash table
* @param data
* @param keyLen
* @return
*/
void *taosHashGetKey(void *data, size_t* keyLen);
/**
* return the payload data with the specified key(reference number added)
@ -258,8 +195,20 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen);
*/
void taosHashRelease(SHashObj *pHashObj, void *p);
/**
*
* @param pHashObj
* @param fp
*/
void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
/**
*
* @param pHashObj
* @param fp
*/
void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp);
#ifdef __cplusplus
}
#endif

56
include/util/tlog.h
View File

@ -22,9 +22,29 @@
extern "C" {
#endif
typedef enum {
DEBUG_FATAL = 1,
DEBUG_ERROR = 1,
DEBUG_WARN = 2,
DEBUG_INFO = 2,
DEBUG_DEBUG = 4,
DEBUG_TRACE = 8,
DEBUG_DUMP = 16,
DEBUG_SCREEN = 64,
DEBUG_FILE = 128
} ELogLevel;
typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content);
extern bool tsLogEmbedded;
extern bool tsAsyncLog;
extern int32_t tsNumOfLogLines;
extern int32_t tsLogKeepDays;
extern LogFp tsLogFp;
extern int64_t tsNumOfErrorLogs;
extern int64_t tsNumOfInfoLogs;
extern int64_t tsNumOfDebugLogs;
extern int64_t tsNumOfTraceLogs;
extern int32_t dDebugFlag;
extern int32_t vDebugFlag;
extern int32_t mDebugFlag;
@ -40,45 +60,33 @@ extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag;
extern int32_t fsDebugFlag;
#define DEBUG_FATAL 1U
#define DEBUG_ERROR DEBUG_FATAL
#define DEBUG_WARN 2U
#define DEBUG_INFO DEBUG_WARN
#define DEBUG_DEBUG 4U
#define DEBUG_TRACE 8U
#define DEBUG_DUMP 16U
#define DEBUG_SCREEN 64U
#define DEBUG_FILE 128U
int32_t taosInitLog(const char *logName, int32_t maxFiles);
void taosCloseLog();
void taosResetLog();
void taosSetAllDebugFlag(int32_t flag);
void taosDumpData(uint8_t *msg, int32_t len);
void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...)
void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...)
#ifdef __GNUC__
__attribute__((format(printf, 3, 4)))
__attribute__((format(printf, 4, 5)))
#endif
;
void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...)
void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...)
#ifdef __GNUC__
__attribute__((format(printf, 3, 4)))
__attribute__((format(printf, 4, 5)))
#endif
;
extern int8_t tscEmbeddedInUtil;
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", DEBUG_INFO, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }}
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); }
#define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
#ifdef __cplusplus
}

2
include/util/tpagedbuf.h
View File

@ -53,7 +53,7 @@ typedef struct SDiskbasedBufStatis {
* @param handle
* @return
*/
int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir);
/**
*

32
source/client/inc/clientInt.h
View File

@ -20,12 +20,12 @@
extern "C" {
#endif
#include "tcommon.h"
#include "parser.h"
#include "query.h"
#include "taos.h"
#include "tcommon.h"
#include "tdatablock.h"
#include "tdef.h"
#include "tep.h"
#include "thash.h"
#include "tlist.h"
#include "tmsg.h"
@ -47,12 +47,12 @@ extern "C" {
typedef struct SAppInstInfo SAppInstInfo;
typedef struct SHbConnInfo {
typedef struct {
void* param;
SClientHbReq* req;
} SHbConnInfo;
typedef struct SAppHbMgr {
typedef struct {
char* key;
// statistics
int32_t reportCnt;
@ -68,11 +68,11 @@ typedef struct SAppHbMgr {
SHashObj* connInfo; // hash<SClientHbKey, SHbConnInfo>
} SAppHbMgr;
typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req);
typedef struct SClientHbMgr {
typedef struct {
int8_t inited;
// ctl
int8_t threadStop;
@ -108,13 +108,13 @@ typedef struct SHeartBeatInfo {
} SHeartBeatInfo;
struct SAppInstInfo {
int64_t numOfConns;
SCorEpSet mgmtEp;
SInstanceSummary summary;
SList* pConnList; // STscObj linked list
int64_t clusterId;
void* pTransporter;
struct SAppHbMgr* pAppHbMgr;
int64_t numOfConns;
SCorEpSet mgmtEp;
SInstanceSummary summary;
SList* pConnList; // STscObj linked list
int64_t clusterId;
void* pTransporter;
SAppHbMgr* pAppHbMgr;
};
typedef struct SAppInfo {
@ -141,10 +141,6 @@ typedef struct STscObj {
SAppInstInfo* pAppInfo;
} STscObj;
typedef struct SMqConsumer {
STscObj* pTscObj;
} SMqConsumer;
typedef struct SReqResultInfo {
const char* pRspMsg;
const char* pData;
@ -172,7 +168,7 @@ typedef struct SRequestSendRecvBody {
SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed.
SDataBuf requestMsg;
int64_t queryJob; // query job, created according to sql query DAG.
struct SQueryDag* pDag; // the query dag, generated according to the sql statement.
struct SQueryDag* pDag; // the query dag, generated according to the sql statement.
SReqResultInfo resInfo;
} SRequestSendRecvBody;

14
source/client/inc/clientLog.h
View File

@ -22,13 +22,13 @@ extern "C" {
#include "tlog.h"
#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus
}

40
source/client/src/clientHb.c
View File

@ -23,7 +23,7 @@ static SClientHbMgr clientHbMgr = {0};
static int32_t hbCreateThread();
static void hbStopThread();
static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; }
static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; }
static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
int32_t code = 0;
@ -104,7 +104,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
return TSDB_CODE_SUCCESS;
}
static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
if (NULL == info) {
tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType);
@ -163,7 +163,7 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRs
return TSDB_CODE_SUCCESS;
}
static int32_t hbMqAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) {
static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) {
static int32_t emptyRspNum = 0;
if (code != 0) {
tfree(param);
@ -226,7 +226,11 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
db->vgVersion = htonl(db->vgVersion);
}
SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = sizeof(SDbVgVersion) * dbNum, .value = dbs};
SKv kv = {
.key = HEARTBEAT_KEY_DBINFO,
.valueLen = sizeof(SDbVgVersion) * dbNum,
.value = dbs,
};
tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen);
@ -256,7 +260,11 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
stb->tversion = htons(stb->tversion);
}
SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs};
SKv kv = {
.key = HEARTBEAT_KEY_STBINFO,
.valueLen = sizeof(SSTableMetaVersion) * stbNum,
.value = stbs,
};
tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen);
@ -288,7 +296,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
return TSDB_CODE_SUCCESS;
}
int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) {}
int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; }
void hbMgrInitMqHbHandle() {
clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle;
@ -396,7 +404,7 @@ static void *hbThreadFunc(void *param) {
free(buf);
break;
}
pInfo->fp = hbMqAsyncCallBack;
pInfo->fp = hbAsyncCallBack;
pInfo->msgInfo.pData = buf;
pInfo->msgInfo.len = tlen;
pInfo->msgType = TDMT_MND_HEARTBEAT;
@ -448,7 +456,6 @@ static void hbStopThread() {
}
SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
/*return NULL;*/
hbMgrInit();
SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr));
if (pAppHbMgr == NULL) {
@ -473,7 +480,8 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
free(pAppHbMgr);
return NULL;
}
pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq;
taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq);
// init getInfoFunc
pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
@ -506,7 +514,6 @@ void appHbMgrCleanup(void) {
}
int hbMgrInit() {
/*return 0;*/
// init once
int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1);
if (old == 1) return 0;
@ -524,7 +531,7 @@ int hbMgrInit() {
}
void hbMgrCleanUp() {
return;
#if 0
hbStopThread();
// destroy all appHbMgr
@ -537,6 +544,7 @@ void hbMgrCleanUp() {
pthread_mutex_unlock(&clientHbMgr.lock);
clientHbMgr.appHbMgrs = NULL;
#endif
}
int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) {
@ -563,9 +571,11 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *
}
int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) {
/*return 0;*/
SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY};
SHbConnInfo info = {0};
SClientHbKey connKey = {
.connId = connId,
.hbType = HEARTBEAT_TYPE_QUERY,
};
SHbConnInfo info = {0};
switch (hbType) {
case HEARTBEAT_TYPE_QUERY: {
@ -586,7 +596,6 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3
}
void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
/*return;*/
int32_t code = 0;
code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey));
@ -598,7 +607,6 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen,
int32_t valueLen) {
return 0;
// find req by connection id
SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
ASSERT(pReq != NULL);

2
source/client/src/clientImpl.c
View File

@ -4,8 +4,8 @@
#include "parser.h"
#include "planner.h"
#include "scheduler.h"
#include "tdatablock.h"
#include "tdef.h"
#include "tep.h"
#include "tglobal.h"
#include "tmsgtype.h"
#include "tpagedbuf.h"

378
source/client/src/tmq.c
View File

@ -20,8 +20,8 @@
#include "parser.h"
#include "planner.h"
#include "scheduler.h"
#include "tdatablock.h"
#include "tdef.h"
#include "tep.h"
#include "tglobal.h"
#include "tmsgtype.h"
#include "tpagedbuf.h"
@ -59,6 +59,7 @@ struct tmq_t {
char groupId[256];
char clientId[256];
int8_t autoCommit;
int8_t inWaiting;
int64_t consumerId;
int32_t epoch;
int32_t resetOffsetCfg;
@ -66,9 +67,12 @@ struct tmq_t {
STscObj* pTscObj;
tmq_commit_cb* commit_cb;
int32_t nextTopicIdx;
int32_t waitingRequest;
int32_t readyRequest;
SArray* clientTopics; // SArray<SMqClientTopic>
STaosQueue* mqueue; // queue of tmq_message_t
STaosQall* qall;
tsem_t rspSem;
// stat
int64_t pollCnt;
};
@ -117,10 +121,12 @@ typedef struct {
} SMqAskEpCbParam;
typedef struct {
tmq_t* tmq;
SMqClientVg* pVg;
int32_t epoch;
tsem_t rspSem;
tmq_t* tmq;
SMqClientVg* pVg;
int32_t epoch;
tsem_t rspSem;
tmq_message_t** msg;
int32_t sync;
} SMqPollCbParam;
typedef struct {
@ -196,6 +202,26 @@ int32_t tmq_list_append(tmq_list_t* ptr, const char* src) {
return 0;
}
void tmqClearUnhandleMsg(tmq_t* tmq) {
tmq_message_t* msg;
while (1) {
taosGetQitem(tmq->qall, (void**)&msg);
if (msg)
taosFreeQitem(msg);
else
break;
}
taosReadAllQitems(tmq->mqueue, tmq->qall);
while (1) {
taosGetQitem(tmq->qall, (void**)&msg);
if (msg)
taosFreeQitem(msg);
else
break;
}
}
int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
pParam->rspErr = code;
@ -205,16 +231,11 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
if (pParam->tmq->commit_cb) {
pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL);
}
if (!pParam->async)
tsem_post(&pParam->rspSem);
else {
tsem_destroy(&pParam->rspSem);
free(param);
pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL, NULL);
}
if (!pParam->async) tsem_post(&pParam->rspSem);
return 0;
}
@ -240,9 +261,12 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
return NULL;
}
pTmq->pTscObj = (STscObj*)conn;
pTmq->inWaiting = 0;
pTmq->status = 0;
pTmq->pollCnt = 0;
pTmq->epoch = 0;
pTmq->waitingRequest = 0;
pTmq->readyRequest = 0;
// set conf
strcpy(pTmq->clientId, conf->clientId);
strcpy(pTmq->groupId, conf->groupId);
@ -250,6 +274,8 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
pTmq->commit_cb = conf->commit_cb;
pTmq->resetOffsetCfg = conf->resetOffset;
tsem_init(&pTmq->rspSem, 0, 0);
pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
@ -315,6 +341,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
}
pParam->tmq = tmq;
tsem_init(&pParam->rspSem, 0, 0);
pParam->async = async;
pRequest->body.requestMsg = (SDataBuf){
.pData = buf,
@ -335,6 +362,9 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
resp = pParam->rspErr;
}
tsem_destroy(&pParam->rspSem);
free(pParam);
if (pArray) {
taosArrayDestroy(pArray);
}
@ -576,7 +606,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return 0;
SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message;
SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
return pRsp->skipLogNum;
}
@ -625,56 +655,74 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
}
int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
printf("recv poll\n");
/*printf("recv poll\n");*/
SMqPollCbParam* pParam = (SMqPollCbParam*)param;
SMqClientVg* pVg = pParam->pVg;
tmq_t* tmq = pParam->tmq;
if (code != 0) {
printf("msg discard\n");
if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
return 0;
goto WRITE_QUEUE_FAIL;
}
int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
if (msgEpoch < tmqEpoch) {
tsem_post(&tmq->rspSem);
printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
return 0;
}
if (msgEpoch != tmqEpoch) {
printf("mismatch rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
} else {
atomic_sub_fetch_32(&tmq->waitingRequest, 1);
}
#if 0
if (pParam->sync == 1) {
/**pParam->msg = malloc(sizeof(tmq_message_t));*/
*pParam->msg = taosAllocateQitem(sizeof(tmq_message_t));
if (*pParam->msg) {
memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &((*pParam->msg)->consumeRsp));
if ((*pParam->msg)->consumeRsp.numOfTopics != 0) {
pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset;
}
taosWriteQitem(tmq->mqueue, *pParam->msg);
tsem_post(&pParam->rspSem);
return 0;
}
tsem_post(&pParam->rspSem);
return -1;
}
#endif
/*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/
tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
if (pRsp == NULL) {
printf("fail\n");
return -1;
goto WRITE_QUEUE_FAIL;
}
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
/*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
if (pRsp->consumeRsp.numOfTopics == 0) {
printf("no data\n");
if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
/*printf("no data\n");*/
taosFreeQitem(pRsp);
return 0;
goto WRITE_QUEUE_FAIL;
}
pRsp->extra = pParam->pVg;
taosWriteQitem(tmq->mqueue, pRsp);
printf("poll in queue\n");
/*pParam->rspMsg = (tmq_message_t*)pRsp;*/
/*pVg->currentOffset = pRsp->consumeRsp.rspOffset;*/
/*printf("rsp offset: %ld\n", rsp.rspOffset);*/
/*printf("-----msg begin----\n");*/
/*printf("\n-----msg end------\n");*/
atomic_add_fetch_32(&tmq->readyRequest, 1);
tsem_post(&tmq->rspSem);
return 0;
WRITE_QUEUE_FAIL:
if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
tsem_post(&tmq->rspSem);
return code;
}
bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
@ -711,81 +759,94 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
tmq_t* tmq = pParam->tmq;
if (code != 0) {
printf("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
if (pParam->sync) {
tsem_post(&pParam->rspSem);
}
return 0;
goto END;
}
tscDebug("tmq ask ep cb called");
// tmq's epoch is monotomically increase,
// so it's safe to discard any old epoch msg.
// epoch will only increase when received newer epoch ep msg
SMqRspHead* head = pMsg->pData;
int32_t epoch = atomic_load_32(&tmq->epoch);
if (head->epoch <= epoch) {
goto END;
}
if (pParam->sync) {
SMqRspHead* head = pMsg->pData;
SMqCMGetSubEpRsp rsp;
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
/*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
/*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
int32_t epoch = atomic_load_32(&tmq->epoch);
if (head->epoch > epoch && tmqUpdateEp(tmq, head->epoch, &rsp)) {
if (tmqUpdateEp(tmq, head->epoch, &rsp)) {
atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY);
}
tsem_post(&pParam->rspSem);
tDeleteSMqCMGetSubEpRsp(&rsp);
} else {
tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = -1;
goto END;
}
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp);
taosWriteQitem(tmq->mqueue, pRsp);
tsem_post(&tmq->rspSem);
}
return 0;
END:
if (pParam->sync) {
tsem_post(&pParam->rspSem);
}
return code;
}
int32_t tmqAskEp(tmq_t* tmq, bool sync) {
printf("ask ep sync %d\n", sync);
int32_t tlen = sizeof(SMqCMGetSubEpReq);
SMqCMGetSubEpReq* buf = malloc(tlen);
if (buf == NULL) {
SMqCMGetSubEpReq* req = malloc(tlen);
if (req == NULL) {
tscError("failed to malloc get subscribe ep buf");
goto END;
return -1;
}
buf->consumerId = htobe64(tmq->consumerId);
buf->epoch = htonl(tmq->epoch);
strcpy(buf->cgroup, tmq->groupId);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP);
if (pRequest == NULL) {
tscError("failed to malloc subscribe ep request");
goto END;
}
pRequest->body.requestMsg = (SDataBuf){
.pData = buf,
.len = tlen,
.handle = NULL,
};
req->consumerId = htobe64(tmq->consumerId);
req->epoch = htonl(tmq->epoch);
strcpy(req->cgroup, tmq->groupId);
SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam));
if (pParam == NULL) {
tscError("failed to malloc subscribe param");
goto END;
free(req);
return -1;
}
pParam->tmq = tmq;
pParam->sync = sync;
tsem_init(&pParam->rspSem, 0, 0);
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
tsem_destroy(&pParam->rspSem);
free(pParam);
free(req);
return -1;
}
sendInfo->msgInfo = (SDataBuf){
.pData = req,
.len = tlen,
.handle = NULL,
};
sendInfo->requestId = generateRequestId();
sendInfo->requestObjRefId = 0;
sendInfo->param = pParam;
sendInfo->fp = tmqAskEpCb;
sendInfo->msgType = TDMT_MND_GET_SUB_EP;
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
END:
if (sync) tsem_wait(&pParam->rspSem);
return 0;
}
@ -804,6 +865,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j);
if (pVg->vgId == pOffset->vgId) {
pVg->currentOffset = pOffset->offset;
tmqClearUnhandleMsg(tmq);
return TMQ_RESP_ERR__SUCCESS;
}
}
@ -812,7 +874,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
return TMQ_RESP_ERR__FAIL;
}
SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) {
SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
int64_t reqOffset;
if (pVg->currentOffset >= 0) {
reqOffset = pVg->currentOffset;
@ -832,7 +894,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie
strcpy(pReq->topic, pTopic->topicName);
strcpy(pReq->cgroup, tmq->groupId);
pReq->blockingTime = blocking_time;
pReq->blockingTime = blockingTime;
pReq->consumerId = tmq->consumerId;
pReq->epoch = tmq->epoch;
pReq->currentOffset = reqOffset;
@ -842,28 +904,76 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie
return pReq;
}
void tmqClearUnhandleMsg(tmq_t* tmq) {
tmq_message_t* msg;
while (1) {
taosGetQitem(tmq->qall, (void**)&msg);
if (msg)
taosFreeQitem(msg);
else
break;
}
tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
tmq_message_t* msg = NULL;
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
/*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/
/*continue;*/
/*}*/
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem
return NULL;
}
taosReadAllQitems(tmq->mqueue, tmq->qall);
while (1) {
taosGetQitem(tmq->qall, (void**)&msg);
if (msg)
taosFreeQitem(msg);
else
break;
SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam));
if (pParam == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem
return NULL;
}
pParam->tmq = tmq;
pParam->pVg = pVg;
pParam->epoch = tmq->epoch;
pParam->sync = 1;
pParam->msg = &msg;
tsem_init(&pParam->rspSem, 0, 0);
SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
return NULL;
}
sendInfo->msgInfo = (SDataBuf){
.pData = pReq,
.len = sizeof(SMqConsumeReq),
.handle = NULL,
};
sendInfo->requestId = generateRequestId();
sendInfo->requestObjRefId = 0;
sendInfo->param = pParam;
sendInfo->fp = tmqPollCb;
sendInfo->msgType = TDMT_VND_CONSUME;
int64_t transporterId = 0;
/*printf("send poll\n");*/
atomic_add_fetch_32(&tmq->waitingRequest, 1);
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
pVg->pollCnt++;
tmq->pollCnt++;
tsem_wait(&pParam->rspSem);
tmq_message_t* nmsg = NULL;
while (1) {
taosReadQitem(tmq->mqueue, (void**)&nmsg);
if (nmsg == NULL) continue;
while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) {
taosReadQitem(tmq->mqueue, (void**)&nmsg);
}
return nmsg;
}
}
}
return NULL;
}
int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
printf("call poll\n");
/*printf("call poll\n");*/
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
@ -875,32 +985,44 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem
tsem_post(&tmq->rspSem);
return -1;
}
SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam));
if (param == NULL) {
SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam));
if (pParam == NULL) {
free(pReq);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem
tsem_post(&tmq->rspSem);
return -1;
}
param->tmq = tmq;
param->pVg = pVg;
param->epoch = tmq->epoch;
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){
pParam->tmq = tmq;
pParam->pVg = pVg;
pParam->epoch = tmq->epoch;
pParam->sync = 0;
SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
free(pReq);
free(pParam);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
tsem_post(&tmq->rspSem);
return -1;
}
sendInfo->msgInfo = (SDataBuf){
.pData = pReq,
.len = sizeof(SMqConsumeReq),
.handle = NULL,
};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestId = generateRequestId();
sendInfo->requestObjRefId = 0;
sendInfo->param = param;
sendInfo->param = pParam;
sendInfo->fp = tmqPollCb;
sendInfo->msgType = TDMT_VND_CONSUME;
int64_t transporterId = 0;
printf("send poll\n");
/*printf("send poll\n");*/
atomic_add_fetch_32(&tmq->waitingRequest, 1);
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
pVg->pollCnt++;
tmq->pollCnt++;
@ -912,7 +1034,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
// return
int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) {
if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);
/*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) {
tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp);
tmqClearUnhandleMsg(tmq);
@ -931,23 +1053,26 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
tmq_message_t* rspMsg = NULL;
taosGetQitem(tmq->qall, (void**)&rspMsg);
if (rspMsg == NULL) {
break;
taosReadAllQitems(tmq->mqueue, tmq->qall);
taosGetQitem(tmq->qall, (void**)&rspMsg);
if (rspMsg == NULL) return NULL;
}
if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) {
printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);
atomic_sub_fetch_32(&tmq->readyRequest, 1);
/*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/
if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) {
printf("epoch match\n");
/*printf("epoch match\n");*/
SMqClientVg* pVg = rspMsg->extra;
pVg->currentOffset = rspMsg->consumeRsp.rspOffset;
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
return rspMsg;
} else {
printf("epoch mismatch\n");
/*printf("epoch mismatch\n");*/
taosFreeQitem(rspMsg);
}
} else {
printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);
/*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
bool reset = false;
tmqHandleRes(tmq, rspMsg, &reset);
taosFreeQitem(rspMsg);
@ -957,36 +1082,57 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
}
}
}
return NULL;
}
tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) {
tmq_message_t* rspMsg = NULL;
int64_t startTime = taosGetTimestampMs();
int64_t status = atomic_load_64(&tmq->status);
tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
while (1) {
rspMsg = tmqSyncPollImpl(tmq, blocking_time);
if (rspMsg && rspMsg->consumeRsp.numOfTopics) {
return rspMsg;
}
if (blocking_time != 0) {
int64_t endTime = taosGetTimestampMs();
if (endTime - startTime > blocking_time) {
return NULL;
}
} else
return NULL;
}
}
tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
tmq_message_t* rspMsg = NULL;
tmq_message_t* rspMsg;
int64_t startTime = taosGetTimestampMs();
// TODO: put into another thread or delayed queue
int64_t status = atomic_load_64(&tmq->status);
tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
taosGetQitem(tmq->qall, (void**)&rspMsg);
if (rspMsg == NULL) {
taosReadAllQitems(tmq->mqueue, tmq->qall);
rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
if (rspMsg) {
return rspMsg;
}
tmqHandleAllRsp(tmq, blocking_time, false);
tmqPollImpl(tmq, blocking_time);
while (1) {
/*printf("cycle\n");*/
taosReadAllQitems(tmq->mqueue, tmq->qall);
rspMsg = tmqHandleAllRsp(tmq, blocking_time, true);
tmqPollImpl(tmq, blocking_time);
tsem_wait(&tmq->rspSem);
rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
if (rspMsg) {
return rspMsg;
}
if (blocking_time != 0) {
int64_t endTime = taosGetTimestampMs();
if (endTime - startTime > blocking_time) {
printf("normal exit\n");
return NULL;
}
}
@ -1127,6 +1273,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return;
SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
tDeleteSMqConsumeRsp(pRsp);
/*free(tmq_message);*/
taosFreeQitem(tmq_message);
}
@ -1138,6 +1285,7 @@ const char* tmq_err2str(tmq_resp_err_t err) {
}
return "fail";
}
#if 0
tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) {
tmq_t* pTmq = malloc(sizeof(tmq_t));

2
source/common/src/tep.c → source/common/src/tdatablock.c
View File

@ -14,7 +14,7 @@
*/
#define _DEFAULT_SOURCE
#include "tep.h"
#include "tdatablock.h"
#include "tcompare.h"
#include "tglobal.h"

80
source/common/src/tglobal.c
View File

@ -17,7 +17,7 @@
#include "tglobal.h"
#include "tcompare.h"
#include "tconfig.h"
#include "tep.h"
#include "tdatablock.h"
#include "tlog.h"
SConfig *tsCfg = NULL;
@ -155,21 +155,40 @@ static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t prima
uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
}
static void taosSetTfsCfg(SConfig *pCfg) {
static int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
if (pItem == NULL) return;
memset(tsDataDir, 0, PATH_MAX);
int32_t size = taosArrayGetSize(pItem->array);
if (size <= 0) {
tsDiskCfgNum = 1;
taosAddDataDir(0, pItem->str, 0, 1);
tstrncpy(tsDataDir, pItem->str, PATH_MAX);
if (taosMkDir(tsDataDir) != 0) {
uError("failed to create dataDir:%s since %s", tsDataDir, terrstr());
return -1;
}
} else {
tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS;
for (int32_t index = 0; index < tsDiskCfgNum; ++index) {
SDiskCfg *pCfg = taosArrayGet(pItem->array, index);
memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg));
if (pCfg->level == 0 && pCfg->primary == 1) {
tstrncpy(tsDataDir, pCfg->dir, PATH_MAX);
}
if (taosMkDir(pCfg->dir) != 0) {
uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr());
return -1;
}
}
}
if (tsDataDir[0] == 0) {
uError("datadir not set");
return -1;
}
return 0;
}
struct SConfig *taosGetCfg() {
@ -279,11 +298,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1;
if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1;
if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "totalMemory(MB)", tsTotalMemoryMB, 0, INT32_MAX, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "totalMemory(KB)", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1;
if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1;
if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1;
@ -358,7 +377,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
}
static void taosSetClientCfg(SConfig *pCfg) {
static int32_t taosSetClientCfg(SConfig *pCfg) {
tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN);
tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
@ -375,9 +394,13 @@ static void taosSetClientCfg(SConfig *pCfg) {
snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port);
cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype);
tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX);
taosExpandDir(tsLogDir, tsLogDir, PATH_MAX);
tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX);
taosExpandDir(tsTempDir, tsTempDir, PATH_MAX);
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval;
if (taosMkDir(tsTempDir) != 0) {
uError("failed to create tempDir:%s since %s", tsTempDir, terrstr());
return -1;
}
tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval;
tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
@ -392,6 +415,8 @@ static void taosSetClientCfg(SConfig *pCfg) {
tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
return 0;
}
static void taosSetSystemCfg(SConfig *pCfg) {
@ -404,10 +429,6 @@ static void taosSetSystemCfg(SConfig *pCfg) {
const char *charset = cfgGetItem(pCfg, "charset")->str;
taosSetSystemLocale(locale, charset);
if (tsNumOfCores <= 1) {
tsNumOfCores = 2;
}
bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
taosSetConsoleEcho(enableCore);
@ -415,11 +436,8 @@ static void taosSetSystemCfg(SConfig *pCfg) {
tsVersion = 30000000;
}
static void taosSetServerCfg(SConfig *pCfg) {
tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX);
taosExpandDir(tsDataDir, tsDataDir, PATH_MAX);
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
static int32_t taosSetServerCfg(SConfig *pCfg) {
tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval;
tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
@ -448,6 +466,8 @@ static void taosSetServerCfg(SConfig *pCfg) {
if (tsQueryBufferSize >= 0) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
}
return 0;
}
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
@ -458,10 +478,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
if (pCfg == NULL) return -1;
if (tsc) {
tscEmbeddedInUtil = 0;
tsLogEmbedded = 0;
if (taosAddClientLogCfg(pCfg) != 0) return -1;
} else {
tscEmbeddedInUtil = 1;
tsLogEmbedded = 1;
if (taosAddClientLogCfg(pCfg) != 0) return -1;
if (taosAddServerLogCfg(pCfg) != 0) return -1;
}
@ -508,8 +528,8 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
tsCfg = cfgInit();
if (tsc) {
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
} else {
if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddServerCfg(tsCfg) != 0) return -1;
@ -532,24 +552,14 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
}
if (tsc) {
taosSetClientCfg(tsCfg);
if (taosSetClientCfg(tsCfg)) return -1;
} else {
taosSetClientCfg(tsCfg);
taosSetServerCfg(tsCfg);
taosSetTfsCfg(tsCfg);
if (taosSetClientCfg(tsCfg)) return -1;
if (taosSetServerCfg(tsCfg)) return -1;
if (taosSetTfsCfg(tsCfg) != 0) return -1;
}
taosSetSystemCfg(tsCfg);
if (taosMkDir(tsTempDir) != 0) {
uError("failed to create dir:%s since %s", tsTempDir, terrstr());
return -1;
}
if (!tsc && taosMkDir(tsDataDir) != 0) {
uError("failed to create dir:%s since %s", tsDataDir, terrstr());
return -1;
}
cfgDumpCfg(tsCfg, tsc, false);
return 0;
}

4
source/common/src/ttszip.c
View File

@ -46,7 +46,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
}
if (!autoDelete) {
remove(pTSBuf->path);
taosRemoveFile(pTSBuf->path);
}
if (NULL == allocResForTSBuf(pTSBuf)) {
@ -178,7 +178,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
if (pTSBuf->autoDelete) {
// ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path);
remove(pTSBuf->path);
taosRemoveFile(pTSBuf->path);
} else {
// tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path);
}

5
source/common/test/commonTests.cpp
View File

@ -1,6 +1,4 @@
#include "tcommon.h"
#include <gtest/gtest.h>
#include <tep.h>
#include <iostream>
#pragma GCC diagnostic push
@ -10,6 +8,9 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "tcommon.h"
#include "tdatablock.h"
#include "tcommon.h"
#include "taos.h"
#include "tvariant.h"
#include "tdef.h"

2
source/common/test/tmsgTest.cpp
View File

@ -1,6 +1,6 @@
#include <iostream>
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "tmsg.h"

6
source/dnode/mgmt/impl/inc/dndEnv.h
View File

@ -31,7 +31,7 @@ typedef struct {
SDnode *pDnode;
STaosQueue *queue;
union {
SQWorkerPool pool;
SQWorkerPool pool;
SWWorkerPool mpool;
};
} SDnodeWorker;
@ -108,6 +108,7 @@ typedef struct {
SHashObj *hash;
int32_t openVnodes;
int32_t totalVnodes;
int32_t masterNum;
SRWLatch latch;
SQWorkerPool queryPool;
SFWorkerPool fetchPool;
@ -137,8 +138,7 @@ typedef struct SDnode {
SStartupReq startup;
} SDnode;
int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo);
int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo);
#ifdef __cplusplus
}

20
source/dnode/mgmt/impl/inc/dndInt.h
View File

@ -23,9 +23,11 @@ extern "C" {
#include "os.h"
#include "cJSON.h"
#include "monitor.h"
#include "tcache.h"
#include "tcrc32c.h"
#include "tep.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "thash.h"
#include "tlockfree.h"
#include "tlog.h"
@ -35,8 +37,6 @@ extern "C" {
#include "tthread.h"
#include "ttime.h"
#include "tworker.h"
#include "tglobal.h"
#include "monitor.h"
#include "dnode.h"
@ -47,14 +47,12 @@ extern "C" {
#include "vnode.h"
#include "tfs.h"
extern int32_t dDebugFlag;
#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }}
#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }}
#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }}
#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", 255, __VA_ARGS__); }}
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;

1
source/dnode/mgmt/impl/inc/dndMnode.h
View File

@ -34,6 +34,7 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo);
int8_t dndIsMnode(SDnode *pDnode);
#ifdef __cplusplus
}

9
source/dnode/mgmt/impl/src/dndEnv.c
View File

@ -324,4 +324,11 @@ void dndCleanup() {
dInfo("dnode env is cleaned up");
}
int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { return 0; }
int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
pInfo->logdir.size = tsLogSpace.size;
tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
pInfo->tempdir.size = tsTempSpace.size;
return tfsGetMonitorInfo(pDnode->pTfs, pInfo);
}

46
source/dnode/mgmt/impl/src/dndMgmt.c
View File

@ -474,13 +474,36 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
rpcSendResponse(&rpcRsp);
}
static int32_t dndGetBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
pInfo->dnode_id = dndGetDnodeId(pDnode);
tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
return 0;
}
static int32_t dndGetDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { return 0; }
static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
pInfo->uptime = (taosGetTimestampMs() - pDnode->dmgmt.rebootTime) / (86400000.0f);
taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system);
pInfo->cpu_cores = tsNumOfCores;
taosGetProcMemory(&pInfo->mem_engine);
taosGetSysMemory(&pInfo->mem_system);
pInfo->mem_total = tsTotalMemoryKB;
pInfo->disk_engine = 0;
pInfo->disk_used = tsDataSpace.size.used;
pInfo->disk_total = tsDataSpace.size.total;
taosGetBandSpeed(&pInfo->net_in, &pInfo->net_out);
taosGetIOSpeed(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
pInfo->req_select = 0;
pInfo->req_select_rate = 0;
pInfo->req_insert = 0;
pInfo->req_insert_success = 0;
pInfo->req_insert_rate = 0;
pInfo->req_insert_batch = 0;
pInfo->req_insert_batch_success = 0;
pInfo->req_insert_batch_rate = 0;
pInfo->errors = tsNumOfErrorLogs;
pInfo->vnodes_num = pDnode->vmgmt.totalVnodes;
pInfo->masters = pDnode->vmgmt.masterNum;
pInfo->has_mnode = dndIsMnode(pDnode);
}
static void dndSendMonitorReport(SDnode *pDnode) {
if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
@ -490,9 +513,8 @@ static void dndSendMonitorReport(SDnode *pDnode) {
if (pMonitor == NULL) return;
SMonBasicInfo basicInfo = {0};
if (dndGetBasicInfo(pDnode, &basicInfo) == 0) {
monSetBasicInfo(pMonitor, &basicInfo);
}
dndGetMonitorBasicInfo(pDnode, &basicInfo);
monSetBasicInfo(pMonitor, &basicInfo);
SMonClusterInfo clusterInfo = {0};
SMonVgroupInfo vgroupInfo = {0};
@ -504,15 +526,19 @@ static void dndSendMonitorReport(SDnode *pDnode) {
}
SMonDnodeInfo dnodeInfo = {0};
if (dndGetDnodeInfo(pDnode, &dnodeInfo) == 0) {
monSetDnodeInfo(pMonitor, &dnodeInfo);
}
dndGetMonitorDnodeInfo(pDnode, &dnodeInfo);
monSetDnodeInfo(pMonitor, &dnodeInfo);
SMonDiskInfo diskInfo = {0};
if (dndGetDiskInfo(pDnode, &diskInfo) == 0) {
if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
monSetDiskInfo(pMonitor, &diskInfo);
}
taosArrayDestroy(clusterInfo.dnodes);
taosArrayDestroy(clusterInfo.mnodes);
taosArrayDestroy(vgroupInfo.vgroups);
taosArrayDestroy(diskInfo.datadirs);
monSendReport(pMonitor);
monCleanupMonitorInfo(pMonitor);
}

7
source/dnode/mgmt/impl/src/dndMnode.c
View File

@ -639,4 +639,11 @@ int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SM
int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
dndReleaseMnode(pDnode, pMnode);
return code;
}
int8_t dndIsMnode(SDnode *pDnode) {
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL) return 0;
dndReleaseMnode(pDnode, pMnode);
return 1;
}

9
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -17,6 +17,7 @@
#include "dndVnodes.h"
#include "dndMgmt.h"
#include "dndTransport.h"
#include "sync.h"
typedef struct {
int32_t vgId;
@ -85,7 +86,7 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
if (pVnode == NULL) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
} else {
@ -979,6 +980,8 @@ void dndCleanupVnodes(SDnode *pDnode) {
void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
int32_t totalVnodes = 0;
int32_t masterNum = 0;
taosRLockLatch(&pMgmt->latch);
@ -993,8 +996,12 @@ void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) {
vnodeGetLoad(pVnode->pImpl, &vload);
taosArrayPush(pLoads, &vload);
totalVnodes++;
if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++;
pIter = taosHashIterate(pMgmt->hash, pIter);
}
taosRUnLockLatch(&pMgmt->latch);
pMgmt->totalVnodes = totalVnodes;
pMgmt->masterNum = masterNum;
}

2
source/dnode/mgmt/impl/test/sut/src/client.cpp
View File

@ -14,7 +14,7 @@
*/
#include "sut.h"
#include "tep.h"
#include "tdatablock.h"
static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
TestClient* client = (TestClient*)parent;

2
source/dnode/mgmt/impl/test/sut/src/sut.cpp
View File

@ -28,7 +28,7 @@ void Testbase::InitLog(const char* path) {
wDebugFlag = 0;
sDebugFlag = 0;
tsdbDebugFlag = 0;
tscEmbeddedInUtil = 1;
tsLogEmbedded = 1;
tsAsyncLog = 0;
taosRemoveDir(path);

40
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -34,46 +34,6 @@
extern "C" {
#endif
extern int32_t mDebugFlag;
// mnode log function
#define mFatal(...) \
{ \
if (mDebugFlag & DEBUG_FATAL) { \
taosPrintLog("MND FATAL ", 255, __VA_ARGS__); \
} \
}
#define mError(...) \
{ \
if (mDebugFlag & DEBUG_ERROR) { \
taosPrintLog("MND ERROR ", 255, __VA_ARGS__); \
} \
}
#define mWarn(...) \
{ \
if (mDebugFlag & DEBUG_WARN) { \
taosPrintLog("MND WARN ", 255, __VA_ARGS__); \
} \
}
#define mInfo(...) \
{ \
if (mDebugFlag & DEBUG_INFO) { \
taosPrintLog("MND ", 255, __VA_ARGS__); \
} \
}
#define mDebug(...) \
{ \
if (mDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
} \
}
#define mTrace(...) \
{ \
if (mDebugFlag & DEBUG_TRACE) { \
taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
} \
}
typedef enum {
MND_AUTH_ACCT_START = 0,
MND_AUTH_ACCT_USER,

17
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -20,7 +20,7 @@
#include "sdb.h"
#include "tcache.h"
#include "tep.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tqueue.h"
#include "ttime.h"
@ -31,6 +31,13 @@
extern "C" {
#endif
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg);
typedef int32_t (*MndInitFp)(SMnode *pMnode);
typedef void (*MndCleanupFp)(SMnode *pMnode);
@ -85,6 +92,11 @@ typedef struct {
ESyncState state;
} SSyncMgmt;
typedef struct {
int64_t expireTimeMS;
int64_t timeseriesAllowed;
} SGrantInfo;
typedef struct SMnode {
int32_t dnodeId;
int64_t clusterId;
@ -105,6 +117,7 @@ typedef struct SMnode {
STelemMgmt telemMgmt;
SSyncMgmt syncMgmt;
SHashObj *infosMeta;
SGrantInfo grant;
MndMsgFp msgFp[TDMT_MAX];
SendReqToDnodeFp sendReqToDnodeFp;
SendReqToMnodeFp sendReqToMnodeFp;
@ -120,7 +133,7 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
uint64_t mndGenerateUid(char *name, int32_t len);
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
#ifdef __cplusplus
}

1
source/dnode/mnode/impl/inc/mndProfile.h
View File

@ -24,6 +24,7 @@ extern "C" {
int32_t mndInitProfile(SMnode *pMnode);
void mndCleanupProfile(SMnode *pMnode);
int32_t mndGetNumOfConnections(SMnode *pMnode);
#ifdef __cplusplus
}

4
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -99,11 +99,15 @@ void mndUpdateMnodeRole(SMnode *pMnode) {
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
ESyncState lastRole = pObj->role;
if (pObj->id == 1) {
pObj->role = TAOS_SYNC_STATE_LEADER;
} else {
pObj->role = TAOS_SYNC_STATE_CANDIDATE;
}
if (pObj->role != lastRole) {
pObj->roleTime = taosGetTimestampMs();
}
sdbRelease(pSdb, pObj);
}

70
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -48,7 +48,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
static void mndFreeConn(SConnObj *pConn);
static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId);
static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn);
static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter);
static void mndCancelGetNextConn(SMnode *pMnode, void *pIter);
static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq);
static int32_t mndProcessConnectReq(SMnodeMsg *pReq);
@ -158,27 +158,23 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) {
taosCacheRelease(pMgmt->cache, (void **)&pConn, false);
}
static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
*pConn = NULL;
pIter = taosHashIterate(pMgmt->cache->pHashTable, pIter);
if (pIter == NULL) return NULL;
SCacheDataNode **pNode = pIter;
if (pNode == NULL || *pNode == NULL) {
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
return NULL;
void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) {
SConnObj* pConn = NULL;
bool hasNext = taosCacheIterNext(pIter);
if (hasNext) {
size_t dataLen = 0;
pConn = taosCacheIterGetData(pIter, &dataLen);
} else {
taosCacheDestroyIter(pIter);
}
*pConn = (SConnObj *)((*pNode)->data);
return pIter;
return pConn;
}
static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
if (pIter != NULL) {
taosCacheDestroyIter(pIter);
}
}
static int32_t mndProcessConnectReq(SMnodeMsg *pReq) {
@ -376,8 +372,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
int32_t rspLen = 0;
mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv);
SKv kv1 = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv1);
}
break;
}
@ -386,8 +382,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
int32_t rspLen = 0;
mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv);
SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv1);
}
break;
}
@ -638,7 +634,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable);
pShow->numOfRows = taosCacheGetNumOfObj(pMgmt->cache);
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbName, mndShowStr(pShow->type));
@ -653,8 +649,13 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in
char *pWrite;
char ipStr[TSDB_IPv4ADDR_LEN + 6];
if (pShow->pIter == NULL) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
pShow->pIter = taosCacheCreateIter(pMgmt->cache);
}
while (numOfRows < rows) {
pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn);
pConn = mndGetNextConn(pMnode, pShow->pIter);
if (pConn == NULL) break;
cols = 0;
@ -823,19 +824,24 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
void *pIter;
char str[TSDB_IPv4ADDR_LEN + 6] = {0};
if (pShow->pIter == NULL) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
pShow->pIter = taosCacheCreateIter(pMgmt->cache);
}
while (numOfRows < rows) {
pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn);
pConn = mndGetNextConn(pMnode, pShow->pIter);
if (pConn == NULL) {
pShow->pIter = pIter;
pShow->pIter = NULL;
break;
}
if (numOfRows + pConn->numOfQueries >= rows) {
mndCancelGetNextConn(pMnode, pIter);
taosCacheDestroyIter(pShow->pIter);
pShow->pIter = NULL;
break;
}
pShow->pIter = pIter;
for (int32_t i = 0; i < pConn->numOfQueries; ++i) {
SQueryDesc *pDesc = pConn->pQueries + i;
cols = 0;
@ -913,6 +919,12 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
}
static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
if (pIter != NULL) {
taosCacheDestroyIter(pIter);
}
}
int32_t mndGetNumOfConnections(SMnode *pMnode) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
return taosCacheGetNumOfObj(pMgmt->cache);
}

19
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -18,15 +18,15 @@
#include "mndCluster.h"
#include "mndSync.h"
#include "tbuffer.h"
#include "tjson.h"
#include "thttp.h"
#include "tjson.h"
#define TELEMETRY_SERVER "telemetry.taosdata.com"
#define TELEMETRY_PORT 80
static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) {
SMnodeLoad load = {0};
if (mndGetLoad(pMnode, &load) != 0) return;
mndGetLoad(pMnode, &load);
tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode);
tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode);
@ -52,23 +52,20 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) {
tjsonAddStringToObject(pJson, "instanceId", clusterName);
tjsonAddDoubleToObject(pJson, "reportVersion", 1);
if (taosGetOsReleaseName(tmp, sizeof(tmp))) {
if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) {
tjsonAddStringToObject(pJson, "os", tmp);
}
int32_t numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores)) {
float numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) {
tjsonAddStringToObject(pJson, "cpuModel", tmp);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores);
} else {
tjsonAddDoubleToObject(pJson, "numOfCpu", taosGetCpuCores());
tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores);
}
uint64_t memoryKB = 0;
if (taosGetTotalSysMemoryKB(&memoryKB)) {
snprintf(tmp, sizeof(tmp), "%" PRIu64 " kB", memoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
}
snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
tjsonAddStringToObject(pJson, "memory", tmp);
tjsonAddStringToObject(pJson, "version", version);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo);

149
source/dnode/mnode/impl/src/mnode.c
View File

@ -359,6 +359,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
return NULL;
}
mndUpdateMnodeRole(pMnode);
mDebug("mnode open successfully ");
return pMnode;
}
@ -385,26 +386,6 @@ void mndDestroy(const char *path) {
mDebug("mnode is destroyed");
}
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
pLoad->numOfDnode = 0;
pLoad->numOfMnode = 0;
pLoad->numOfVgroup = 0;
pLoad->numOfDatabase = 0;
pLoad->numOfSuperTable = 0;
pLoad->numOfChildTable = 0;
pLoad->numOfColumn = 0;
pLoad->totalPoints = 0;
pLoad->totalStorage = 0;
pLoad->compStorage = 0;
return 0;
}
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) {
return 0;
}
SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg));
if (pMsg == NULL) {
@ -523,3 +504,131 @@ uint64_t mndGenerateUid(char *name, int32_t len) {
}
} while (true);
}
void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
memset(pLoad, 0, sizeof(SMnodeLoad));
SSdb *pSdb = pMnode->pSdb;
pLoad->numOfDnode = sdbGetSize(pSdb, SDB_DNODE);
pLoad->numOfMnode = sdbGetSize(pSdb, SDB_MNODE);
pLoad->numOfVgroup = sdbGetSize(pSdb, SDB_VGROUP);
pLoad->numOfDatabase = sdbGetSize(pSdb, SDB_DB);
pLoad->numOfSuperTable = sdbGetSize(pSdb, SDB_STB);
void *pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
pLoad->numOfChildTable += pVgroup->numOfTables;
pLoad->numOfColumn += pVgroup->numOfTimeSeries;
pLoad->totalPoints += pVgroup->pointsWritten;
pLoad->totalStorage += pVgroup->totalStorage;
pLoad->compStorage += pVgroup->compStorage;
sdbRelease(pSdb, pVgroup);
}
}
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) {
if (!mndIsMaster(pMnode)) return -1;
SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs();
pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc));
pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc));
pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL) {
return -1;
}
// cluster info
tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version));
pClusterInfo->monitor_interval = tsMonitorInterval;
pClusterInfo->connections_total = mndGetNumOfConnections(pMnode);
void *pIter = NULL;
while (1) {
SDnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SMonDnodeDesc desc = {0};
desc.dnode_id = pObj->id;
tstrncpy(desc.dnode_ep, pObj->ep, sizeof(desc.dnode_ep));
if (mndIsDnodeOnline(pMnode, pObj, ms)) {
tstrncpy(desc.status, "ready", sizeof(desc.status));
} else {
tstrncpy(desc.status, "offline", sizeof(desc.status));
}
taosArrayPush(pClusterInfo->dnodes, &desc);
sdbRelease(pSdb, pObj);
}
pIter = NULL;
while (1) {
SMnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SMonMnodeDesc desc = {0};
desc.mnode_id = pObj->id;
tstrncpy(desc.mnode_ep, pObj->pDnode->ep, sizeof(desc.mnode_ep));
tstrncpy(desc.role, mndGetRoleStr(pObj->role), sizeof(desc.role));
taosArrayPush(pClusterInfo->mnodes, &desc);
sdbRelease(pSdb, pObj);
if (pObj->role == TAOS_SYNC_STATE_LEADER) {
pClusterInfo->first_ep_dnode_id = pObj->id;
tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
pClusterInfo->master_uptime = (ms - pObj->roleTime) / (86400000.0f);
}
}
// vgroup info
pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
pClusterInfo->vgroups_total++;
SMonVgroupDesc desc = {0};
desc.vgroup_id = pVgroup->vgId;
strncpy(desc.database_name, pVgroup->dbName, sizeof(desc.database_name));
desc.tables_num = pVgroup->numOfTables;
pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries;
tstrncpy(desc.status, "unsynced", sizeof(desc.status));
for (int32_t i = 0; i < pVgroup->replica; ++i) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
SMonVnodeDesc *pVnDesc = &desc.vnodes[i];
pVnDesc->dnode_id = pVgid->dnodeId;
tstrncpy(pVnDesc->vnode_role, mndGetRoleStr(pVgid->role), sizeof(pVnDesc->vnode_role));
if (pVgid->role == TAOS_SYNC_STATE_LEADER) {
tstrncpy(desc.status, "ready", sizeof(desc.status));
pClusterInfo->vgroups_alive++;
}
if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_CANDIDATE) {
pClusterInfo->vnodes_alive++;
}
pClusterInfo->vnodes_total++;
}
taosArrayPush(pVgroupInfo->vgroups, &desc);
sdbRelease(pSdb, pVgroup);
}
// grant info
pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f;
pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed;
if (pMnode->grant.expireTimeMS == 0) {
pGrantInfo->expire_time = INT32_MAX;
pGrantInfo->timeseries_total = INT32_MAX;
}
return 0;
}

12
source/dnode/mnode/sdb/inc/sdbInt.h
View File

@ -28,12 +28,12 @@
extern "C" {
#endif
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }}
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }}
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }}
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", 255, __VA_ARGS__); }}
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
#define SDB_MAX_SIZE (32 * 1024)

12
source/dnode/vnode/inc/meta.h
View File

@ -24,8 +24,8 @@
extern "C" {
#endif
#define META_SUPER_TABLE TD_SUPER_TABLE
#define META_CHILD_TABLE TD_CHILD_TABLE
#define META_SUPER_TABLE TD_SUPER_TABLE
#define META_CHILD_TABLE TD_CHILD_TABLE
#define META_NORMAL_TABLE TD_NORMAL_TABLE
// Types exported
@ -50,14 +50,14 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid);
int metaCommit(SMeta *pMeta);
// For Query
STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
void metaCloseTbCursor(SMTbCursor *pTbCur);
char * metaTbCursorNext(SMTbCursor *pTbCur);
char *metaTbCursorNext(SMTbCursor *pTbCur);
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid);
void metaCloseCtbCurosr(SMCtbCursor *pCtbCur);

21
source/dnode/vnode/inc/vnode.h
View File

@ -213,6 +213,10 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA
//}
static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) {
if (pHandle->tbIdHash) {
taosHashClear(pHandle->tbIdHash);
}
pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (pHandle->tbIdHash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -227,6 +231,23 @@ static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const S
return 0;
}
static FORCE_INLINE int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) {
if (pHandle->tbIdHash == NULL) {
pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (pHandle->tbIdHash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
for (int i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i);
taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
}
return 0;
}
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle);
int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo);

2
source/dnode/vnode/src/inc/metaDef.h
View File

@ -76,4 +76,4 @@ struct SMeta {
}
#endif
#endif /*_TD_META_DEF_H_*/
#endif /*_TD_META_DEF_H_*/

62
source/dnode/vnode/src/inc/tqInt.h
View File

@ -25,43 +25,41 @@
extern "C" {
#endif
extern int32_t tqDebugFlag;
#define tqFatal(...) \
{ \
if (tqDebugFlag & DEBUG_FATAL) { \
taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); \
} \
#define tqFatal(...) \
{ \
if (tqDebugFlag & DEBUG_FATAL) { \
taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
} \
}
#define tqError(...) \
{ \
if (tqDebugFlag & DEBUG_ERROR) { \
taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); \
} \
#define tqError(...) \
{ \
if (tqDebugFlag & DEBUG_ERROR) { \
taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \
}
#define tqWarn(...) \
{ \
if (tqDebugFlag & DEBUG_WARN) { \
taosPrintLog("TQ WARN ", 255, __VA_ARGS__); \
} \
#define tqWarn(...) \
{ \
if (tqDebugFlag & DEBUG_WARN) { \
taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \
}
#define tqInfo(...) \
{ \
if (tqDebugFlag & DEBUG_INFO) { \
taosPrintLog("TQ ", 255, __VA_ARGS__); \
} \
#define tqInfo(...) \
{ \
if (tqDebugFlag & DEBUG_INFO) { \
taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); \
} \
}
#define tqDebug(...) \
{ \
if (tqDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \
} \
#define tqDebug(...) \
{ \
if (tqDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \
} \
}
#define tqTrace(...) \
{ \
if (tqDebugFlag & DEBUG_TRACE) { \
taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \
} \
#define tqTrace(...) \
{ \
if (tqDebugFlag & DEBUG_TRACE) { \
taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); \
} \
}
#define TQ_BUFFER_SIZE 8

12
source/dnode/vnode/src/inc/tsdbLog.h
View File

@ -20,11 +20,11 @@
extern int32_t tsdbDebugFlag;
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0)
#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0)
#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#endif /* _TD_TSDB_LOG_H_ */

63
source/dnode/vnode/src/inc/vnd.h
View File

@ -88,44 +88,41 @@ int vnodeScheduleTask(SVnodeTask* task);
int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq);
void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq);
// For Log
extern int32_t vDebugFlag;
#define vFatal(...) \
do { \
if (vDebugFlag & DEBUG_FATAL) { \
taosPrintLog("VND FATAL ", 255, __VA_ARGS__); \
} \
#define vFatal(...) \
do { \
if (vDebugFlag & DEBUG_FATAL) { \
taosPrintLog("VND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
} \
} while (0)
#define vError(...) \
do { \
if (vDebugFlag & DEBUG_ERROR) { \
taosPrintLog("VND ERROR ", 255, __VA_ARGS__); \
} \
#define vError(...) \
do { \
if (vDebugFlag & DEBUG_ERROR) { \
taosPrintLog("VND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \
} while (0)
#define vWarn(...) \
do { \
if (vDebugFlag & DEBUG_WARN) { \
taosPrintLog("VND WARN ", 255, __VA_ARGS__); \
} \
#define vWarn(...) \
do { \
if (vDebugFlag & DEBUG_WARN) { \
taosPrintLog("VND WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \
} while (0)
#define vInfo(...) \
do { \
if (vDebugFlag & DEBUG_INFO) { \
taosPrintLog("VND ", 255, __VA_ARGS__); \
} \
#define vInfo(...) \
do { \
if (vDebugFlag & DEBUG_INFO) { \
taosPrintLog("VND ", DEBUG_INFO, 255, __VA_ARGS__); \
} \
} while (0)
#define vDebug(...) \
do { \
if (vDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \
} \
#define vDebug(...) \
do { \
if (vDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("VND ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define vTrace(...) \
do { \
if (vDebugFlag & DEBUG_TRACE) { \
taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \
} \
#define vTrace(...) \
do { \
if (vDebugFlag & DEBUG_TRACE) { \
taosPrintLog("VND ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); \
} \
} while (0)
// vnodeCfg.h

34
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -62,10 +62,10 @@ static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *
static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg);
static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
static void metaClearTbCfg(STbCfg *pTbCfg);
static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
static void * metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
static void metaDBWLock(SMetaDB *pDB);
static void metaDBRLock(SMetaDB *pDB);
static void metaDBULock(SMetaDB *pDB);
@ -142,7 +142,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
tb_uid_t uid;
char buf[512];
char buf1[512];
void * pBuf;
void *pBuf;
DBT key1, value1;
DBT key2, value2;
SSchema *pSchema = NULL;
@ -394,7 +394,7 @@ static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey
static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) {
STbCfg *pTbCfg = (STbCfg *)(pValue->app_data);
DBT * pDbt;
DBT *pDbt;
if (pTbCfg->type == META_CHILD_TABLE) {
// pDbt = calloc(2, sizeof(DBT));
@ -479,7 +479,7 @@ static void metaClearTbCfg(STbCfg *pTbCfg) {
/* ------------------------ FOR QUERY ------------------------ */
STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
STbCfg * pTbCfg = NULL;
STbCfg *pTbCfg = NULL;
SMetaDB *pDB = pMeta->pDB;
DBT key = {0};
DBT value = {0};
@ -509,7 +509,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
}
STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
STbCfg * pTbCfg = NULL;
STbCfg *pTbCfg = NULL;
SMetaDB *pDB = pMeta->pDB;
DBT key = {0};
DBT pkey = {0};
@ -543,10 +543,10 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
uint32_t nCols;
SSchemaWrapper *pSW = NULL;
SMetaDB * pDB = pMeta->pDB;
SMetaDB *pDB = pMeta->pDB;
int ret;
void * pBuf;
SSchema * pSchema;
void *pBuf;
SSchema *pSchema;
SSchemaKey schemaKey = {uid, sver, 0};
DBT key = {0};
DBT value = {0};
@ -578,7 +578,7 @@ struct SMTbCursor {
SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
SMTbCursor *pTbCur = NULL;
SMetaDB * pDB = pMeta->pDB;
SMetaDB *pDB = pMeta->pDB;
pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur));
if (pTbCur == NULL) {
@ -609,7 +609,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
DBT key = {0};
DBT value = {0};
STbCfg tbCfg;
void * pBuf;
void *pBuf;
for (;;) {
if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) {
@ -631,10 +631,10 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
STSchemaBuilder sb;
STSchema * pTSchema = NULL;
SSchema * pSchema;
STSchema *pTSchema = NULL;
SSchema *pSchema;
SSchemaWrapper *pSW;
STbCfg * pTbCfg;
STbCfg *pTbCfg;
tb_uid_t quid;
pTbCfg = metaGetTbInfoByUid(pMeta, uid);
@ -662,13 +662,13 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
}
struct SMCtbCursor {
DBC * pCur;
DBC *pCur;
tb_uid_t suid;
};
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
SMCtbCursor *pCtbCur = NULL;
SMetaDB * pDB = pMeta->pDB;
SMetaDB *pDB = pMeta->pDB;
int ret;
pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur));
@ -700,7 +700,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
DBT skey = {0};
DBT pkey = {0};
DBT pval = {0};
void * pBuf;
void *pBuf;
STbCfg tbCfg;
// Set key

4
source/dnode/vnode/src/tq/tq.c
View File

@ -72,6 +72,8 @@ void tqClose(STQ* pTq) {
}
int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
// if waiting
// memcpy and send msg to fetch thread
// TODO: add reference
// if handle waiting, launch query and response to consumer
//
@ -210,7 +212,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeReq* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId;
int64_t fetchOffset;
/*int64_t blockingTime = pReq->blockingTime;*/
int64_t blockingTime = pReq->blockingTime;
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
fetchOffset = 0;

2
source/dnode/vnode/src/tq/tqMetaStore.c
View File

@ -90,7 +90,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
char name[pathLen + 10];
strcpy(name, path);
if (taosDirExist(name) != 0 && taosMkDir(name) != 0) {
if (!taosDirExist(name) && taosMkDir(name) != 0) {
terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR;
tqError("failed to create dir:%s since %s ", name, terrstr());
}

6
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -580,7 +580,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
SDFile *pRSmadF = TSDB_READ_SMAD_FILE(&(pCommith->readh));
SDFile *pWSmadF = TSDB_COMMIT_SMAD_FILE(pCommith);
if (access(TSDB_FILE_FULL_NAME(pRSmadF), F_OK) != 0) {
if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmadF))) {
tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF));
tsdbInitDFile(pRepo, pWSmadF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAD);
@ -614,7 +614,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
SDFile *pRSmalF = TSDB_READ_SMAL_FILE(&(pCommith->readh));
SDFile *pWSmalF = TSDB_COMMIT_SMAL_FILE(pCommith);
if ((pCommith->isLFileSame) && access(TSDB_FILE_FULL_NAME(pRSmalF), F_OK) == 0) {
if ((pCommith->isLFileSame) && taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmalF))) {
tsdbInitDFileEx(pWSmalF, pRSmalF);
if (tsdbOpenDFile(pWSmalF, O_RDWR) < 0) {
tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF),
@ -1054,7 +1054,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
// pfs->metaCacheComp = NULL;
// } else {
// // remove meta.tmp file
// remove(mf.f.aname);
// taosRemoveFile(mf.f.aname);
// taosHashCleanup(pfs->metaCacheComp);
// pfs->metaCacheComp = NULL;
// }

10
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -314,7 +314,7 @@ int tsdbOpenFS(STsdb *pRepo) {
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current);
tsdbGetRtnSnap(pRepo, &pRepo->rtn);
if (access(current, F_OK) == 0) {
if (taosCheckExistFile(current)) {
if (tsdbOpenFSFromCurrent(pRepo) < 0) {
tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno));
return -1;
@ -439,7 +439,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile);
remove(tfname);
taosRemoveFile(tfname);
return -1;
}
@ -447,7 +447,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (fsheader.len > 0) {
if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
taosCloseFile(&pFile);
remove(tfname);
taosRemoveFile(tfname);
return -1;
}
@ -458,7 +458,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile);
(void)remove(tfname);
(void)taosRemoveFile(tfname);
taosTZfree(pBuf);
return -1;
}
@ -468,7 +468,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (taosFsyncFile(pFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile);
remove(tfname);
taosRemoveFile(tfname);
taosTZfree(pBuf);
return -1;
}

15
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -443,25 +443,24 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) {
}
static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
struct stat dfstat;
SDFile df;
tsdbInitDFileEx(&df, pDFile);
if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) {
if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pDFile))) {
tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pDFile));
// pRepo->state |= TSDB_STATE_BAD_DATA;
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
return 0;
}
if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) {
int64_t file_size = 0;
if (taosStatFile(TSDB_FILE_FULL_NAME(&df), &file_size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (pDFile->info.size < dfstat.st_size) {
if (pDFile->info.size < file_size) {
// if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) {
return -1;
@ -480,10 +479,10 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
tsdbCloseDFile(&df);
tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
dfstat.st_size, pDFile->info.size);
} else if (pDFile->info.size > dfstat.st_size) {
file_size, pDFile->info.size);
} else if (pDFile->info.size > file_size) {
tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size);
REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), file_size, pDFile->info.size);
// pRepo->state |= TSDB_STATE_BAD_DATA;
TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;

1
source/libs/CMakeLists.txt
View File

@ -1,4 +1,3 @@
add_definitions("-D ALLOW_FORBID_FUNC")
add_subdirectory(transport)
add_subdirectory(sync)
add_subdirectory(tdb)

60
source/libs/catalog/src/catalog.c
View File

@ -168,7 +168,7 @@ void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
}
int32_t colNum = c->numOfColumns + c->numOfTags;
for (int32_t i = 0; i < colNum; ++i) {
SSchema *s = &p->schema[i];
@ -190,7 +190,7 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
dbCache = (SCtgDBCache *)pIter;
taosHashGetKey(dbCache, (void **)&dbFName, &len);
taosHashGetKey((void **)&dbFName, &len);
int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
@ -204,9 +204,9 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
if (dbCache->vgInfo->vgHash) {
vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
}
}
}
ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
pIter = taosHashIterate(dbHash, pIter);
@ -222,7 +222,7 @@ void ctgDbgShowClusterCache(SCatalog* pCtg) {
}
ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg);
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));
ctgDbgShowDBCache(pCtg, pCtg->dbCache);
@ -306,9 +306,9 @@ int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId)
return TSDB_CODE_SUCCESS;
_return:
tfree(action.data);
CTG_RET(code);
CTG_RET(code);
}
@ -336,9 +336,9 @@ int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId
return TSDB_CODE_SUCCESS;
_return:
tfree(action.data);
CTG_RET(code);
CTG_RET(code);
}
@ -366,9 +366,9 @@ int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId
return TSDB_CODE_SUCCESS;
_return:
tfree(action.data);
CTG_RET(code);
CTG_RET(code);
}
@ -657,9 +657,9 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
return TSDB_CODE_SUCCESS;
}
size_t sz = 0;
int32_t sz = 0;
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
STableMeta *tbMeta = taosHashGetCloneExt(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), NULL, (void **)pTableMeta, &sz);
int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), (void **)pTableMeta, &sz);
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (NULL == *pTableMeta) {
@ -673,8 +673,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
if (dbId) {
*dbId = dbCache->dbId;
}
tbMeta = *pTableMeta;
STableMeta* tbMeta = *pTableMeta;
if (tbMeta->tableType != TSDB_CHILD_TABLE) {
ctgReleaseDBCache(pCtg, dbCache);
@ -1076,7 +1076,7 @@ _return:
}
int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t compare) {
int16_t widx = abs(id % mgmt->slotNum);
int16_t widx = labs(id % mgmt->slotNum);
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
@ -1238,7 +1238,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) {
void *pIter = taosHashIterate(cache->stbCache, NULL);
while (pIter) {
uint64_t *suid = NULL;
taosHashGetKey(pIter, (void **)&suid, NULL);
suid = taosHashGetKey(pIter, NULL);
if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionCompare)) {
ctgDebug("stb removed from rent, suid:%"PRIx64, *suid);
@ -1397,7 +1397,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid);
ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare);
}
@ -1437,7 +1437,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) {
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, meta->suid);
ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
@ -1475,7 +1475,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
int32_t *vgId = NULL;
void *pIter = taosHashIterate(src->vgHash, NULL);
while (pIter) {
taosHashGetKey(pIter, (void **)&vgId, NULL);
vgId = taosHashGetKey(pIter, NULL);
if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) {
qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize);
@ -1635,7 +1635,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps,
if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) {
CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist));
}
if (0 == exist) {
CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput));
@ -1723,9 +1723,9 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
tbType = (*pTableMeta)->tableType;
suid = (*pTableMeta)->suid;
tfree(*pTableMeta);
tfree(*pTableMeta);
}
if (CTG_FLAG_IS_UNKNOWN_STB(flag)) {
CTG_FLAG_SET_STB(flag, tbType);
}
@ -1950,21 +1950,21 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) {
ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName);
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) {
if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
}
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
_return:
tfree(msg);
CTG_RET(code);
}
@ -2458,7 +2458,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm
CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList));
} else {
int32_t vgId = tbMeta->vgId;
if (NULL == taosHashGetClone(vgHash, &vgId, sizeof(vgId), &vgroupInfo)) {
if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) {
ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}

10
source/libs/catalog/test/catalogTests.cpp
View File

@ -14,9 +14,7 @@
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#include "os.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
@ -24,16 +22,18 @@
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wformat"
#include <addr_any.h>
#include "addr_any.h"
#include "os.h"
#include "tglobal.h"
#include "catalog.h"
#include "catalogInt.h"
#include "stub.h"
#include "taos.h"
#include "tdatablock.h"
#include "tdef.h"
#include "tep.h"
#include "trpc.h"
#include "tvariant.h"
#include "catalogInt.h"
namespace {

2
source/libs/executor/inc/executil.h
View File

@ -68,7 +68,7 @@ typedef struct SResultRow {
} SResultRow;
typedef struct SResultRowInfo {
SResultRow *pCurResult; // current active result row info
SList* pRows;
SResultRow** pResult; // result list
// int16_t type:8; // data type for hash key
int32_t size; // number of result set

50
source/libs/executor/inc/executorimpl.h
View File

@ -15,12 +15,12 @@
#ifndef TDENGINE_EXECUTORIMPL_H
#define TDENGINE_EXECUTORIMPL_H
#include "tsort.h"
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "tsort.h"
#include "tcommon.h"
#include "tlosertree.h"
#include "ttszip.h"
@ -157,6 +157,13 @@ typedef struct STaskCostInfo {
SHashObj* operatorProfResults; // map<operator_type, SQueryProfEvent>
} STaskCostInfo;
typedef struct SOperatorCostInfo {
uint64_t openCost;
uint64_t execCost;
uint64_t totalRows;
uint64_t totalBytes;
} SOperatorCostInfo;
typedef struct {
int64_t vgroupLimit;
int64_t ts;
@ -233,9 +240,9 @@ typedef struct STaskAttr {
SArray* pUdfInfo; // no need to free
} STaskAttr;
typedef int32_t (*__optr_prepare_fn_t)(void* param);
typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup);
typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num);
typedef int32_t (*__optr_open_fn_t)(void* param);
typedef SSDataBlock* (*__optr_fn_t)(void* param, bool* newgroup);
typedef void (*__optr_close_fn_t)(void* param, int32_t num);
struct SOperatorInfo;
@ -306,21 +313,21 @@ enum {
};
typedef struct SOperatorInfo {
uint8_t operatorType;
bool blockingOptr; // block operator or not
uint8_t status; // denote if current operator is completed
int32_t numOfOutput; // number of columns of the current operator results
char* name; // name, used to show the query execution plan
void* info; // extension attribution
SExprInfo* pExpr;
STaskRuntimeEnv* pRuntimeEnv; // todo remove it
SExecTaskInfo* pTaskInfo;
uint8_t operatorType;
bool blockingOptr; // block operator or not
uint8_t status; // denote if current operator is completed
int32_t numOfOutput; // number of columns of the current operator results
char* name; // name, used to show the query execution plan
void* info; // extension attribution
SExprInfo* pExpr;
STaskRuntimeEnv* pRuntimeEnv; // todo remove it
SExecTaskInfo* pTaskInfo;
struct SOperatorInfo** pDownstream; // downstram pointer list
int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator
__optr_prepare_fn_t prepareFn;
__operator_fn_t exec;
__optr_cleanup_fn_t cleanupFn;
__optr_open_fn_t openFn;
__optr_fn_t nextDataFn;
__optr_close_fn_t closeFn;
} SOperatorInfo;
typedef struct {
@ -479,9 +486,6 @@ typedef struct SAggOperatorInfo {
typedef struct SProjectOperatorInfo {
SOptrBasicInfo binfo;
int32_t bufCapacity;
uint32_t seed;
SSDataBlock* existDataBlock;
} SProjectOperatorInfo;
@ -615,10 +619,10 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
int32_t numOfOutput);
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
SExprInfo* pExpr, int32_t numOfOutput);
@ -654,8 +658,6 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOf
SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
// SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
// SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
// SSDataBlock* doSLimit(void* param, bool* newgroup);
// int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);

104
source/libs/executor/inc/tsimplehash.h Normal file
View File

@ -0,0 +1,104 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSIMPLEHASH_H
#define TDENGINE_TSIMPLEHASH_H
#include "tarray.h"
#include "tlockfree.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef uint32_t (*_hash_fn_t)(const char *, uint32_t);
typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_free_fn_t)(void *);
typedef struct SSHashObj SSHashObj;
/**
* init the hash table
*
* @param capacity initial capacity of the hash table
* @param fn hash function to generate the hash value
* @return
*/
SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen);
/**
* return the size of hash table
* @param pHashObj
* @return
*/
int32_t tSimpleHashGetSize(const SSHashObj *pHashObj);
/**
* put element into hash table, if the element with the same key exists, update it
* @param pHashObj
* @param key
* @param data
* @return
*/
int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data);
/**
* return the payload data with the specified key
*
* @param pHashObj
* @param key
* @return
*/
void *tSimpleHashGet(SSHashObj *pHashObj, const void *key);
/**
* remove item with the specified key
* @param pHashObj
* @param key
* @param keyLen
*/
int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key);
/**
* Clear the hash table.
* @param pHashObj
*/
void tSimpleHashClear(SSHashObj *pHashObj);
/**
* Clean up hash table and release all allocated resources.
* @param handle
*/
void tSimpleHashCleanup(SSHashObj *pHashObj);
/**
* Get the hash table size
* @param pHashObj
* @return
*/
size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj);
/**
* Get the corresponding key information for a given data in hash table
* @param data
* @param keyLen
* @return
*/
void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSIMPLEHASH_H

8
source/libs/executor/src/executor.c
View File

@ -95,17 +95,17 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
}
int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo* )tinfo;
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
// traverse to the streamscan node to add this table id
SOperatorInfo* pInfo = pTaskInfo->pRoot;
while(pInfo->operatorType != OP_StreamScan) {
while (pInfo->operatorType != OP_StreamScan) {
pInfo = pInfo->pDownstream[0];
}
SStreamBlockScanInfo* pScanInfo = pInfo->info;
if (isAdd) {
int32_t code = tqReadHandleSetTbUidList(pScanInfo->readerHandle, tableIdList);
int32_t code = tqReadHandleAddTbUidList(pScanInfo->readerHandle, tableIdList);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -114,4 +114,4 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
}
return TSDB_CODE_SUCCESS;
}
}

41
source/libs/executor/src/executorMain.c
View File

@ -158,7 +158,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
int64_t st = 0;
st = taosGetTimestampUs();
*pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup);
*pRes = pTaskInfo->pRoot->nextDataFn(pTaskInfo->pRoot, &newgroup);
uint64_t el = (taosGetTimestampUs() - st);
pTaskInfo->cost.elapsedTime += el;
@ -179,13 +179,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
return pTaskInfo->code;
}
void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) {
SQInfo* pQInfo = (SQInfo*) qinfo;
assert(pQInfo != NULL);
return pQInfo->rspContext;
}
int32_t qKillTask(qTaskInfo_t qinfo) {
SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo;
@ -221,7 +214,7 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
int32_t qIsTaskCompleted(qTaskInfo_t qinfo) {
SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo;
if (pTaskInfo == NULL /*|| !isValidQInfo(pTaskInfo)*/) {
if (pTaskInfo == NULL) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
@ -235,33 +228,3 @@ void qDestroyTask(qTaskInfo_t qTaskHandle) {
queryCostStatis(pTaskInfo); // print the query cost summary
doDestroyTask(pTaskInfo);
}
#if 0
//kill by qid
int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) {
int32_t error = TSDB_CODE_SUCCESS;
void** handle = qAcquireTask(pMgmt, qId);
if(handle == NULL) return terrno;
SQInfo* pQInfo = (SQInfo*)(*handle);
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
qWarn("%s be killed(no memory commit).", pQInfo->qId);
setTaskKilled(pQInfo);
// wait query stop
int32_t loop = 0;
while (pQInfo->owner != 0) {
taosMsleep(waitMs);
if(loop++ > waitCount){
error = TSDB_CODE_FAILED;
break;
}
}
qReleaseTask(pMgmt, (void **)&handle, true);
return error;
}
#endif

175
source/libs/executor/src/executorimpl.c
View File

@ -15,12 +15,12 @@
#include "os.h"
#include "tep.h"
#include "tsort.h"
#include "texception.h"
#include "parser.h"
#include "tdatablock.h"
#include "texception.h"
#include "tglobal.h"
#include "tmsg.h"
#include "tsort.h"
#include "ttime.h"
#include "executorimpl.h"
@ -2215,10 +2215,10 @@ static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) {
if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) {
SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0);
if (pColInfoData) {
FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor
if (f) {
fclose(f);
*(FILE **)pColInfoData->pData = NULL;
TdFilePtr pFile = *(TdFilePtr *)pColInfoData->pData; // TODO refactor
if (pFile != NULL) {
taosCloseFile(&pFile);
*(TdFilePtr *)pColInfoData->pData = NULL;
}
}
}
@ -4619,7 +4619,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize);
int32_t TENMB = 1024*1024*10;
int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, "/tmp");
int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, "", "/tmp");
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -5281,7 +5281,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->numOfOutput = size;
pOperator->exec = doLoadRemoteData;
pOperator->nextDataFn = doLoadRemoteData;
pOperator->pTaskInfo = pTaskInfo;
#if 1
@ -5361,7 +5361,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->exec = doTableScan;
pOperator->nextDataFn = doTableScan;
pOperator->pTaskInfo = pTaskInfo;
return pOperator;
@ -5386,7 +5386,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim
pOperator->info = pInfo;
pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doTableScanImpl;
pOperator->nextDataFn = doTableScanImpl;
return pOperator;
}
@ -5410,7 +5410,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
// pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols;
pOperator->exec = doBlockInfoScan;
pOperator->nextDataFn = doBlockInfoScan;
return pOperator;
}
@ -5452,7 +5452,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->exec = doStreamBlockScan;
pOperator->nextDataFn = doStreamBlockScan;
pOperator->pTaskInfo = pTaskInfo;
return pOperator;
}
@ -5663,7 +5663,7 @@ SSDataBlock* loadNextDataBlock(void* param) {
SOperatorInfo* pOperator = (SOperatorInfo*) param;
bool newgroup = false;
return pOperator->exec(pOperator, &newgroup);
return pOperator->nextDataFn(pOperator, &newgroup);
}
static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char **buf, int32_t rowIndex) {
@ -5983,8 +5983,8 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t
pOperator->pExpr = exprArrayDup(pExprInfo);
pOperator->pTaskInfo = pTaskInfo;
pOperator->exec = doSortedMerge;
pOperator->cleanupFn = destroySortedMergeOperatorInfo;
pOperator->nextDataFn = doSortedMerge;
pOperator->closeFn = destroySortedMergeOperatorInfo;
code = appendDownstream(pOperator, downstream, numOfDownstream);
if (code != TSDB_CODE_SUCCESS) {
@ -6079,8 +6079,8 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->exec = doSort;
pOperator->cleanupFn = destroyOrderOperatorInfo;
pOperator->nextDataFn = doSort;
pOperator->closeFn = destroyOrderOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -6105,7 +6105,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6155,7 +6155,7 @@ static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6224,7 +6224,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);
updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows);
updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows);
projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput);
@ -6241,7 +6241,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
// The downstream exec may change the value of the newgroup, so use a local variable instead.
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6274,7 +6274,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);
updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows);
updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows);
projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput);
pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput);
@ -6299,7 +6299,7 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) {
SSDataBlock* pBlock = NULL;
while (1) {
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6350,7 +6350,7 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) {
while (1) {
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock *pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
SSDataBlock *pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6393,7 +6393,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6453,7 +6453,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6516,7 +6516,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6571,7 +6571,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6706,7 +6706,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) {
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6768,7 +6768,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
break;
@ -6821,7 +6821,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
break;
@ -6906,7 +6906,7 @@ static SSDataBlock* doFill(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (*newgroup) {
@ -6979,8 +6979,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) {
return;
}
if (pOperator->cleanupFn != NULL) {
pOperator->cleanupFn(pOperator->info, pOperator->numOfOutput);
if (pOperator->closeFn != NULL) {
pOperator->closeFn(pOperator->info, pOperator->numOfOutput);
}
if (pOperator->pDownstream != NULL) {
@ -7067,8 +7067,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE
pOperator->numOfOutput = taosArrayGetSize(pExprInfo);
pOperator->pTaskInfo = pTaskInfo;
pOperator->exec = doAggregate;
pOperator->cleanupFn = destroyAggOperatorInfo;
pOperator->nextDataFn = doAggregate;
pOperator->closeFn = destroyAggOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7164,38 +7164,34 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray
pOperator->pExpr = exprArrayDup(pExprInfo);
pOperator->numOfOutput = numOfOutput;
pOperator->exec = doMultiTableAggregate;
pOperator->cleanupFn = destroyAggOperatorInfo;
pOperator->nextDataFn = doMultiTableAggregate;
pOperator->closeFn = destroyAggOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
}
SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) {
SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo));
pInfo->seed = rand();
pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity;
int32_t numOfRows = 4096;
pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows);
pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize);
SOptrBasicInfo* pBInfo = &pInfo->binfo;
pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity);
pBInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset);
initResultRowInfo(&pBInfo->resultRowInfo, 8);
setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN);
// initResultRowInfo(&pBInfo->resultRowInfo, 8);
// setDefaultOutputBuf_rv(pBInfo, MAIN_SCAN);
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
pOperator->name = "ProjectOperator";
// pOperator->operatorType = OP_Project;
pOperator->operatorType = OP_Project;
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfOutput;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->pExpr = exprArrayDup(pExprInfo);
pOperator->numOfOutput = taosArrayGetSize(pExprInfo);
pOperator->exec = doProjectOperation;
pOperator->cleanupFn = destroyProjectOperatorInfo;
pOperator->nextDataFn = doProjectOperation;
pOperator->closeFn = destroyProjectOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7250,10 +7246,10 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
pOperator->status = OP_IN_EXECUTING;
pOperator->numOfOutput = numOfOutput;
pOperator->pExpr = pExpr;
pOperator->exec = doFilter;
pOperator->nextDataFn = doFilter;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->cleanupFn = destroyConditionOperatorInfo;
pOperator->closeFn = destroyConditionOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7269,7 +7265,7 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn
// pOperator->operatorType = OP_Limit;
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->exec = doLimit;
pOperator->nextDataFn = doLimit;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
int32_t code = appendDownstream(pOperator, &downstream, 1);
@ -7277,24 +7273,18 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn
return pOperator;
}
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo) {
STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
initAggSup(&pInfo->aggSup, pExprInfo);
// todo:
pInfo->order = TSDB_ORDER_ASC;
pInfo->precision = TSDB_TIME_PRECISION_MICRO;
pInfo->win.skey = INT64_MIN;
pInfo->win.ekey = INT64_MAX;
pInfo->interval.intervalUnit = 's';
pInfo->interval.slidingUnit = 's';
pInfo->interval.interval = 1000;
pInfo->interval.sliding = 1000;
pInfo->win = pTaskInfo->window;
pInfo->interval = *pInterval;
int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, 0, "/tmp/");
int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/");
int32_t numOfOutput = taosArrayGetSize(pExprInfo);
pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize);
pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity);
@ -7309,16 +7299,15 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pEx
pOperator->pExpr = exprArrayDup(pExprInfo);
pOperator->pTaskInfo = pTaskInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->numOfOutput = taosArrayGetSize(pExprInfo);
pOperator->info = pInfo;
pOperator->exec = doIntervalAgg;
pOperator->cleanupFn = destroyBasicOperatorInfo;
pOperator->nextDataFn = doIntervalAgg;
pOperator->closeFn = destroyBasicOperatorInfo;
code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
}
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
@ -7336,8 +7325,8 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doAllIntervalAgg;
pOperator->cleanupFn = destroyBasicOperatorInfo;
pOperator->nextDataFn = doAllIntervalAgg;
pOperator->closeFn = destroyBasicOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7360,8 +7349,8 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doStateWindowAgg;
pOperator->cleanupFn = destroyStateWindowOperatorInfo;
pOperator->nextDataFn = doStateWindowAgg;
pOperator->closeFn = destroyStateWindowOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7385,8 +7374,8 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doSessionWindowAgg;
pOperator->cleanupFn = destroySWindowOperatorInfo;
pOperator->nextDataFn = doSessionWindowAgg;
pOperator->closeFn = destroySWindowOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7409,8 +7398,8 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doSTableIntervalAgg;
pOperator->cleanupFn = destroyBasicOperatorInfo;
pOperator->nextDataFn = doSTableIntervalAgg;
pOperator->closeFn = destroyBasicOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7433,8 +7422,8 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doAllSTableIntervalAgg;
pOperator->cleanupFn = destroyBasicOperatorInfo;
pOperator->nextDataFn = doAllSTableIntervalAgg;
pOperator->closeFn = destroyBasicOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
@ -7465,8 +7454,8 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = hashGroupbyAggregate;
pOperator->cleanupFn = destroyGroupbyOperatorInfo;
pOperator->nextDataFn = hashGroupbyAggregate;
pOperator->closeFn = destroyGroupbyOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7504,8 +7493,8 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doFill;
pOperator->cleanupFn = destroySFillOperatorInfo;
pOperator->nextDataFn = doFill;
pOperator->closeFn = destroySFillOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7553,7 +7542,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
// pOperator->exec = doSLimit;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->cleanupFn = destroySlimitOperatorInfo;
pOperator->closeFn = destroySlimitOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7707,11 +7696,11 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->exec = doTagScan;
pOperator->nextDataFn = doTagScan;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfOutput;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->cleanupFn = destroyTagScanOperatorInfo;
pOperator->closeFn = destroyTagScanOperatorInfo;
return pOperator;
}
@ -7777,7 +7766,7 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -7849,9 +7838,9 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = hashDistinct;
pOperator->nextDataFn = hashDistinct;
pOperator->pExpr = pExpr;
pOperator->cleanupFn = destroyDistinctOperatorInfo;
pOperator->closeFn = destroyDistinctOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -8741,10 +8730,8 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type
static int64_t getQuerySupportBufSize(size_t numOfTables) {
size_t s1 = sizeof(STableQueryInfo);
size_t s2 = sizeof(SHashNode);
// size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb
return (int64_t)((s1 + s2) * 1.5 * numOfTables);
return (int64_t)(s1* 1.5 * numOfTables);
}
int32_t checkForQueryBuf(size_t numOfTables) {

7
source/libs/executor/src/tlinearhash.c
View File

@ -29,8 +29,8 @@ typedef struct SLHashBucket {
typedef struct SLHashObj {
SDiskbasedBuf *pBuf;
_hash_fn_t hashFn;
int32_t tuplesPerPage;
SLHashBucket **pBucket; // entry list
int32_t tuplesPerPage;
int32_t numOfAlloc; // number of allocated bucket ptr slot
int32_t bits; // the number of bits used in hash
int32_t numOfBuckets; // the number of buckets
@ -142,7 +142,7 @@ static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket
pBucket->size -= 1;
}
static void doCompressBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) {
static void doTrimBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) {
size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList);
if (numOfPages <= 1) {
return;
@ -253,6 +253,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_
return NULL;
}
// disable compress when flushing to disk
setBufPageCompressOnDisk(pHashObj->pBuf, false);
/**
@ -367,7 +368,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data
releaseBufPage(pHashObj->pBuf, p);
}
doCompressBucketPages(pHashObj, pBucket);
doTrimBucketPages(pHashObj, pBucket);
}
return TSDB_CODE_SUCCESS;

309
source/libs/executor/src/tsimplehash.c Normal file
View File

@ -0,0 +1,309 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tsimplehash.h"
#include "taoserror.h"
#define SHASH_DEFAULT_LOAD_FACTOR 0.75
#define HASH_MAX_CAPACITY (1024*1024*16)
#define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR)
#define GET_SHASH_NODE_KEY(_n, _dl) ((char*)(_n) + sizeof(SHNode) + (_dl))
#define GET_SHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHNode))
#define HASH_INDEX(v, c) ((v) & ((c)-1))
#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR)
#define FREE_HASH_NODE(_n) \
do { \
tfree(_n); \
} while (0);
typedef struct SHNode {
struct SHNode *next;
char data[];
} SHNode;
typedef struct SSHashObj {
SHNode **hashList;
size_t capacity; // number of slots
size_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function
_equal_fn_t equalFp; // equal function
int32_t keyLen;
int32_t dataLen;
} SSHashObj;
static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
int32_t len = MIN(length, HASH_MAX_CAPACITY);
int32_t i = 4;
while (i < len) i = (i << 1u);
return i;
}
SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen) {
ASSERT(fn != NULL);
if (capacity == 0) {
capacity = 4;
}
SSHashObj* pHashObj = (SSHashObj*) calloc(1, sizeof(SSHashObj));
if (pHashObj == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
// the max slots is not defined by user
pHashObj->capacity = taosHashCapacity((int32_t)capacity);
pHashObj->equalFp = memcmp;
pHashObj->hashFp = fn;
ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
pHashObj->keyLen = keyLen;
pHashObj->dataLen = dataLen;
pHashObj->hashList = (SHNode **)calloc(pHashObj->capacity, sizeof(void *));
if (pHashObj->hashList == NULL) {
free(pHashObj);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
return pHashObj;
}
int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) {
if (pHashObj == NULL) {
return 0;
}
return (int32_t)atomic_load_64(&pHashObj->size);
}
static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
SHNode *pNewNode = malloc(sizeof(SHNode) + keyLen + dsize);
if (pNewNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pNewNode->next = NULL;
memcpy(GET_SHASH_NODE_DATA(pNewNode), pData, dsize);
memcpy(GET_SHASH_NODE_KEY(pNewNode, dsize), key, keyLen);
return pNewNode;
}
void taosHashTableResize(SSHashObj *pHashObj) {
if (!HASH_NEED_RESIZE(pHashObj)) {
return;
}
int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
if (newCapacity > HASH_MAX_CAPACITY) {
// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
// pHashObj->capacity, HASH_MAX_CAPACITY);
return;
}
int64_t st = taosGetTimestampUs();
void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity);
if (pNewEntryList == NULL) {
// qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
return;
}
size_t inc = newCapacity - pHashObj->capacity;
memset(pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc);
pHashObj->hashList = pNewEntryList;
pHashObj->capacity = newCapacity;
for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) {
SHNode* pNode = pHashObj->hashList[idx];
SHNode *pNext;
SHNode *pPrev = NULL;
if (pNode == NULL) {
continue;
}
while (pNode != NULL) {
void* key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen);
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->dataLen);
int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity);
pNext = pNode->next;
if (newIdx != idx) {
if (pPrev == NULL) {
pHashObj->hashList[idx] = pNext;
} else {
pPrev->next = pNext;
}
pNode->next = pHashObj->hashList[newIdx];
pHashObj->hashList[newIdx] = pNode;
} else {
pPrev = pNode;
}
pNode = pNext;
}
}
int64_t et = taosGetTimestampUs();
// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity,
// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
}
int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) {
if (pHashObj == NULL || key == NULL) {
return -1;
}
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
// need the resize process, write lock applied
if (SHASH_NEED_RESIZE(pHashObj)) {
taosHashTableResize(pHashObj);
}
int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
SHNode *pNode = pHashObj->hashList[slot];
if (pNode == NULL) {
SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal);
if (pNewNode == NULL) {
return -1;
}
pHashObj->hashList[slot] = pNewNode;
return 0;
}
while (pNode) {
if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) {
break;
}
pNode = pNode->next;
}
if (pNode == NULL) {
SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal);
if (pNewNode == NULL) {
return -1;
}
pNewNode->next = pHashObj->hashList[slot];
pHashObj->hashList[slot] = pNewNode;
atomic_add_fetch_64(&pHashObj->size, 1);
} else { //update data
memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen);
}
return 0;
}
static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, int32_t index) {
SHNode *pNode = pHashObj->hashList[index];
while (pNode) {
if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) {
break;
}
pNode = pNode->next;
}
return pNode;
}
static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) {
return tSimpleHashGetSize(pHashObj) == 0;
}
void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) {
if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL) {
return NULL;
}
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
SHNode *pNode = pHashObj->hashList[slot];
if (pNode == NULL) {
return NULL;
}
char *data = NULL;
pNode = doSearchInEntryList(pHashObj, key, slot);
if (pNode != NULL) {
data = GET_SHASH_NODE_DATA(pNode);
}
return data;
}
int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key) {
// todo
}
void tSimpleHashClear(SSHashObj *pHashObj) {
if (pHashObj == NULL) {
return;
}
SHNode *pNode, *pNext;
for (int32_t i = 0; i < pHashObj->capacity; ++i) {
pNode = pHashObj->hashList[i];
if (pNode == NULL) {
continue;
}
while (pNode) {
pNext = pNode->next;
FREE_HASH_NODE(pNode);
pNode = pNext;
}
}
pHashObj->size = 0;
}
void tSimpleHashCleanup(SSHashObj *pHashObj) {
if (pHashObj == NULL) {
return;
}
tSimpleHashClear(pHashObj);
tfree(pHashObj->hashList);
}
size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) {
if (pHashObj == NULL) {
return 0;
}
return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj);
}
void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) {
int32_t offset = offsetof(SHNode, data);
SHNode *node = data - offset;
if (keyLen != NULL) {
*keyLen = pHashObj->keyLen;
}
return GET_SHASH_NODE_KEY(node, pHashObj->dataLen);
}

4
source/libs/executor/src/tsort.c
View File

@ -16,11 +16,11 @@
#include "tcommon.h"
#include "query.h"
#include "tsort.h"
#include "tep.h"
#include "tdatablock.h"
#include "tdef.h"
#include "tlosertree.h"
#include "tpagedbuf.h"
#include "tsort.h"
#include "tutil.h"
typedef struct STupleHandle {

712
source/libs/executor/test/executorTests.cpp
View File

@ -13,10 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <executorimpl.h>
#include <function.h>
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic push
@ -26,10 +23,13 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "tglobal.h"
#include "executorimpl.h"
#include "function.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "tep.h"
#include "tdatablock.h"
#include "trpc.h"
#include "stub.h"
#include "executor.h"
@ -201,9 +201,9 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_
pOperator->name = "dummyInputOpertor4Test";
if (numOfCols == 1) {
pOperator->exec = getDummyBlock;
pOperator->nextDataFn = getDummyBlock;
} else {
pOperator->exec = get2ColsDummyBlock;
pOperator->nextDataFn = get2ColsDummyBlock;
}
SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo));
@ -222,100 +222,600 @@ int main(int argc, char** argv) {
return RUN_ALL_TESTS();
}
#if 0
TEST(testCase, build_executor_tree_Test) {
const char* msg = "{\n"
"\t\"Id\":\t{\n"
"\t\t\"QueryId\":\t1.3108161807422521e+19,\n"
"\t\t\"TemplateId\":\t0,\n"
"\t\t\"SubplanId\":\t0\n"
"\t},\n"
"\t\"Node\":\t{\n"
"\t\t\"Name\":\t\"TableScan\",\n"
"\t\t\"Targets\":\t[{\n"
"\t\t\t\t\"Base\":\t{\n"
"\t\t\t\t\t\"Schema\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\t\t\"ColId\":\t5000,\n"
"\t\t\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t\t\t},\n"
"\t\t\t\t\t\"Columns\":\t[{\n"
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
"\t\t\t\t\t\t\t\"Info\":\t{\n"
"\t\t\t\t\t\t\t\t\"ColId\":\t1,\n"
"\t\t\t\t\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t\t\t\t\t}\n"
"\t\t\t\t\t\t}],\n"
"\t\t\t\t\t\"InterBytes\":\t0\n"
"\t\t\t\t},\n"
"\t\t\t\t\"Expr\":\t{\n"
"\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\"Column\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\t\t\"ColId\":\t1,\n"
"\t\t\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t\t\t}\n"
"\t\t\t\t}\n"
"\t\t\t}, {\n"
"\t\t\t\t\"Base\":\t{\n"
"\t\t\t\t\t\"Schema\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\t\"ColId\":\t5001,\n"
"\t\t\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t\t\t},\n"
"\t\t\t\t\t\"Columns\":\t[{\n"
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
"\t\t\t\t\t\t\t\"Info\":\t{\n"
"\t\t\t\t\t\t\t\t\"ColId\":\t2,\n"
"\t\t\t\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t\t\t\t\t}\n"
"\t\t\t\t\t\t}],\n"
"\t\t\t\t\t\"InterBytes\":\t0\n"
"\t\t\t\t},\n"
"\t\t\t\t\"Expr\":\t{\n"
"\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\"Column\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\t\"ColId\":\t2,\n"
"\t\t\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t\t\t}\n"
"\t\t\t\t}\n"
"\t\t\t}],\n"
"\t\t\"InputSchema\":\t[{\n"
"\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\"ColId\":\t5000,\n"
"\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t}, {\n"
"\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\"ColId\":\t5001,\n"
"\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t}],\n"
"\t\t\"TableScan\":\t{\n"
"\t\t\t\"TableId\":\t1,\n"
"\t\t\t\"TableType\":\t2,\n"
"\t\t\t\"Flag\":\t0,\n"
"\t\t\t\"Window\":\t{\n"
"\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n"
"\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n"
"\t\t\t}\n"
"\t\t}\n"
"\t},\n"
"\t\"DataSink\":\t{\n"
"\t\t\"Name\":\t\"Dispatch\",\n"
"\t\t\"Dispatch\":\t{\n"
"\t\t}\n"
"\t}\n"
" \"Type\": \"33\",\n"
" \"Name\": \"PhysiProject\",\n"
" \"PhysiProject\": {\n"
" \"OutputDataBlockDesc\": {\n"
" \"Type\": \"19\",\n"
" \"Name\": \"TupleDesc\",\n"
" \"TupleDesc\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"Slots\": [\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"0\",\n"
" \"DataType\": {\n"
" \"Type\": \"9\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"1\",\n"
" \"DataType\": {\n"
" \"Type\": \"4\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"4\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"2\",\n"
" \"DataType\": {\n"
" \"Type\": \"8\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"20\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"3\",\n"
" \"DataType\": {\n"
" \"Type\": \"5\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"4\",\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"5\",\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" },\n"
" \"Children\": [\n"
" {\n"
" \"Type\": \"30\",\n"
" \"Name\": \"PhysiTableScan\",\n"
" \"PhysiTableScan\": {\n"
" \"OutputDataBlockDesc\": {\n"
" \"Type\": \"19\",\n"
" \"Name\": \"TupleDesc\",\n"
" \"TupleDesc\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"Slots\": [\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"0\",\n"
" \"DataType\": {\n"
" \"Type\": \"9\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"1\",\n"
" \"DataType\": {\n"
" \"Type\": \"4\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"4\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"2\",\n"
" \"DataType\": {\n"
" \"Type\": \"8\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"20\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"3\",\n"
" \"DataType\": {\n"
" \"Type\": \"5\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"4\",\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"5\",\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" },\n"
" \"ScanCols\": [\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"9\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"ts\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"1\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"ts\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"1\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"4\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"4\"\n"
" },\n"
" \"AliasName\": \"c1\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"2\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c1\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"2\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"8\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"20\"\n"
" },\n"
" \"AliasName\": \"c2\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"3\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c2\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"3\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"5\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c3\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"4\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c3\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"4\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c4\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"5\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c4\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"5\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c5\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"6\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c5\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" }\n"
" ],\n"
" \"TableId\": \"1\",\n"
" \"TableType\": \"3\",\n"
" \"ScanOrder\": \"1\",\n"
" \"ScanCount\": \"1\",\n"
" \"ReverseScanCount\": \"0\",\n"
" \"ScanFlag\": \"0\",\n"
" \"StartKey\": \"-9223372036854775808\",\n"
" \"EndKey\": \"9223372036854775807\"\n"
" }\n"
" }\n"
" ],\n"
" \"Projections\": [\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"0\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"9\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"ts\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"1\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"ts\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"1\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"4\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"4\"\n"
" },\n"
" \"AliasName\": \"c1\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"2\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c1\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"1\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"2\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"8\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"20\"\n"
" },\n"
" \"AliasName\": \"c2\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"3\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c2\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"2\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"3\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"5\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c3\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"4\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c3\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"3\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"4\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c4\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"5\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c4\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"4\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"5\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c5\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"6\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c5\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"5\"\n"
" }\n"
" }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
"}";
SExecTaskInfo* pTaskInfo = nullptr;
DataSinkHandle sinkHandle = nullptr;
SReadHandle handle = {.reader = reinterpret_cast<void*>(0x1), .meta = reinterpret_cast<void*>(0x1)};
// int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
SSubplan* plan = NULL;
qStringToSubplan(msg, &plan);
int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
}
TEST(testCase, inMem_sort_Test) {
@ -334,10 +834,10 @@ TEST(testCase, inMem_sort_Test) {
exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
taosArrayPush(pExprInfo, &exp1);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal, NULL);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 5, 1000, data_asc, 1), pExprInfo, pOrderVal, NULL);
bool newgroup = false;
SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup);
SSDataBlock* pRes = pOperator->nextDataFn(pOperator, &newgroup);
SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
@ -400,7 +900,7 @@ TEST(testCase, external_sort_Test) {
exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
// taosArrayPush(pExprInfo, &exp1);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal, NULL);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 1500, 1000, data_desc, 1), pExprInfo, pOrderVal, NULL);
bool newgroup = false;
SSDataBlock* pRes = NULL;
@ -412,7 +912,7 @@ TEST(testCase, external_sort_Test) {
while(1) {
int64_t s = taosGetTimestampUs();
pRes = pOperator->exec(pOperator, &newgroup);
pRes = pOperator->nextDataFn(pOperator, &newgroup);
int64_t e = taosGetTimestampUs();
if (t++ == 1) {
@ -435,7 +935,7 @@ TEST(testCase, external_sort_Test) {
int64_t s2 = taosGetTimestampUs();
printf("total:%ld\n", s2 - s1);
pOperator->cleanupFn(pOperator->info, 2);
pOperator->closeFn(pOperator->info, 2);
tfree(exp);
tfree(exp1);
taosArrayDestroy(pExprInfo);
@ -469,7 +969,7 @@ TEST(testCase, sorted_merge_Test) {
int32_t numOfSources = 10;
SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*));
for(int32_t i = 0; i < numOfSources; ++i) {
plist[i] = createDummyOperator(1, 1, 1, data_asc);
plist[i] = createDummyOperator(1, 1, 1, data_asc, 1);
}
SOperatorInfo* pOperator = createSortedMergeOperatorInfo(plist, numOfSources, pExprInfo, pOrderVal, NULL, NULL);
@ -484,7 +984,7 @@ TEST(testCase, sorted_merge_Test) {
while(1) {
int64_t s = taosGetTimestampUs();
pRes = pOperator->exec(pOperator, &newgroup);
pRes = pOperator->nextDataFn(pOperator, &newgroup);
int64_t e = taosGetTimestampUs();
if (t++ == 1) {
@ -507,15 +1007,13 @@ TEST(testCase, sorted_merge_Test) {
int64_t s2 = taosGetTimestampUs();
printf("total:%ld\n", s2 - s1);
pOperator->cleanupFn(pOperator->info, 2);
pOperator->closeFn(pOperator->info, 2);
tfree(exp);
tfree(exp1);
taosArrayDestroy(pExprInfo);
taosArrayDestroy(pOrderVal);
}
#endif
TEST(testCase, time_interval_Operator_Test) {
srand(time(NULL));
@ -548,7 +1046,11 @@ TEST(testCase, time_interval_Operator_Test) {
SOperatorInfo* p = createDummyOperator(1, 1, 2000, data_asc, 2);
SExecTaskInfo ti = {0};
SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &ti);
SInterval interval = {0};
interval.sliding = interval.interval = 1000;
interval.slidingUnit = interval.intervalUnit = 'a';
SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &interval, &ti);
bool newgroup = false;
SSDataBlock* pRes = NULL;
@ -560,7 +1062,7 @@ TEST(testCase, time_interval_Operator_Test) {
while(1) {
int64_t s = taosGetTimestampUs();
pRes = pOperator->exec(pOperator, &newgroup);
pRes = pOperator->nextDataFn(pOperator, &newgroup);
int64_t e = taosGetTimestampUs();
if (t++ == 1) {
@ -583,7 +1085,7 @@ TEST(testCase, time_interval_Operator_Test) {
int64_t s2 = taosGetTimestampUs();
printf("total:%ld\n", s2 - s1);
pOperator->cleanupFn(pOperator->info, 2);
pOperator->closeFn(pOperator->info, 2);
tfree(exp);
tfree(exp1);
taosArrayDestroy(pExprInfo);

4
source/libs/executor/test/sortTests.cpp
View File

@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <executorimpl.h>
#include <gtest/gtest.h>
#include <tglobal.h>
#include <tsort.h>
@ -26,11 +25,12 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "executorimpl.h"
#include "executor.h"
#include "stub.h"
#include "taos.h"
#include "tdatablock.h"
#include "tdef.h"
#include "tep.h"
#include "trpc.h"
#include "tvariant.h"

2
source/libs/function/src/taggfunction.c
View File

@ -28,8 +28,8 @@
#include "tbuffer.h"
#include "tcompression.h"
//#include "queryLog.h"
#include "tdatablock.h"
#include "tudf.h"
#include "tep.h"
#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput))
#define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y)))

2
source/libs/function/src/tpercentile.c
View File

@ -255,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval,
resetSlotInfo(pBucket);
int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, "/tmp");
int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", "/tmp");
if (ret != 0) {
tMemBucketDestroy(pBucket);
return NULL;

62
source/libs/index/inc/indexInt.h
View File

@ -96,7 +96,7 @@ typedef struct SIndexTermQuery {
typedef struct Iterate Iterate;
typedef struct IterateValue {
int8_t type;
int8_t type; // opera type, ADD_VALUE/DELETE_VALUE
char* colVal;
SArray* val;
} IterateValue;
@ -125,41 +125,41 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf);
// int32_t indexSerialKey(ICacheKey* key, char* buf);
// int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
#define indexFatal(...) \
do { \
if (sDebugFlag & DEBUG_FATAL) { \
taosPrintLog("index FATAL ", 255, __VA_ARGS__); \
} \
#define indexFatal(...) \
do { \
if (sDebugFlag & DEBUG_FATAL) { \
taosPrintLog("index FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
} \
} while (0)
#define indexError(...) \
do { \
if (sDebugFlag & DEBUG_ERROR) { \
taosPrintLog("index ERROR ", 255, __VA_ARGS__); \
} \
#define indexError(...) \
do { \
if (sDebugFlag & DEBUG_ERROR) { \
taosPrintLog("index ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \
} while (0)
#define indexWarn(...) \
do { \
if (sDebugFlag & DEBUG_WARN) { \
taosPrintLog("index WARN ", 255, __VA_ARGS__); \
} \
#define indexWarn(...) \
do { \
if (sDebugFlag & DEBUG_WARN) { \
taosPrintLog("index WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \
} while (0)
#define indexInfo(...) \
do { \
if (sDebugFlag & DEBUG_INFO) { \
taosPrintLog("index ", 255, __VA_ARGS__); \
} \
#define indexInfo(...) \
do { \
if (sDebugFlag & DEBUG_INFO) { \
taosPrintLog("index ", DEBUG_INFO, 255, __VA_ARGS__); \
} \
} while (0)
#define indexDebug(...) \
do { \
if (sDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
#define indexDebug(...) \
do { \
if (sDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("index ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define indexTrace(...) \
do { \
if (sDebugFlag & DEBUG_TRACE) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
#define indexTrace(...) \
do { \
if (sDebugFlag & DEBUG_TRACE) { \
taosPrintLog("index ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0)

15
source/libs/index/inc/index_util.h
View File

@ -54,7 +54,22 @@ extern "C" {
* output:[4, 5]
*/
void iIntersection(SArray *interResults, SArray *finalResult);
/* multi sorted result intersection
* input: [1, 2, 4, 5]
* [2, 3, 4, 5]
* [1, 4, 5]
* output:[1, 2, 3, 4, 5]
*/
void iUnion(SArray *interResults, SArray *finalResult);
/* sorted array
* total: [1, 2, 4, 5, 7, 8]
* except: [4, 5]
* return: [1, 2, 7, 8]
*/
void iExcept(SArray *total, SArray *except);
#ifdef __cplusplus
}
#endif

121
source/libs/index/src/index.c
View File

@ -16,6 +16,7 @@
#include "index.h"
#include "indexInt.h"
#include "index_cache.h"
#include "index_comm.h"
#include "index_tfile.h"
#include "index_util.h"
#include "tdef.h"
@ -30,8 +31,18 @@
void* indexQhandle = NULL;
static char JSON_COLUMN[] = "JSON";
#define INDEX_MERGE_ADD_DEL(src, dst, tgt) \
{ \
bool f = false; \
for (int i = 0; i < taosArrayGetSize(src); i++) { \
if (*(uint64_t*)taosArrayGet(src, i) == tgt) { \
f = true; \
} \
} \
if (f == false) { \
taosArrayPush(dst, &tgt); \
} \
}
void indexInit() {
// refactor later
indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index");
@ -52,6 +63,12 @@ typedef struct SIdxColInfo {
int cVersion;
} SIdxColInfo;
typedef struct SIdxTempResult {
SArray* total;
SArray* added;
SArray* deled;
} SIdxTempResult;
static pthread_once_t isInit = PTHREAD_ONCE_INIT;
// static void indexInit();
static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result);
@ -62,8 +79,7 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp
static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch);
// merge cache and tfile by opera type
static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv);
static void indexMergeSameKey(SArray* result, TFileValue* tv);
static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTempResult* helper);
// static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf);
// int32_t indexSerialKey(ICacheKey* key, char* buf);
@ -379,7 +395,6 @@ static void indexInterResultsDestroy(SArray* results) {
static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) {
// refactor, merge interResults into fResults by oType
for (int i = 0; i < taosArrayGetSize(interResults); i--) {
SArray* t = taosArrayGetP(interResults, i);
taosArraySort(t, uidCompare);
@ -398,44 +413,82 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType
return 0;
}
static void indexMergeSameKey(SArray* result, TFileValue* tv) {
int32_t sz = result ? taosArrayGetSize(result) : 0;
SIdxTempResult* sIdxTempResultCreate() {
SIdxTempResult* tr = calloc(1, sizeof(SIdxTempResult));
tr->total = taosArrayInit(4, sizeof(uint64_t));
tr->added = taosArrayInit(4, sizeof(uint64_t));
tr->deled = taosArrayInit(4, sizeof(uint64_t));
return tr;
}
void sIdxTempResultClear(SIdxTempResult* tr) {
if (tr == NULL) {
return;
}
taosArrayClear(tr->total);
taosArrayClear(tr->added);
taosArrayClear(tr->deled);
}
void sIdxTempResultDestroy(SIdxTempResult* tr) {
if (tr == NULL) {
return;
}
taosArrayDestroy(tr->total);
taosArrayDestroy(tr->added);
taosArrayDestroy(tr->deled);
}
static void sIdxTempResultMergeTo(SArray* result, SIdxTempResult* tr) {
taosArraySort(tr->total, uidCompare);
taosArraySort(tr->added, uidCompare);
taosArraySort(tr->deled, uidCompare);
SArray* arrs = taosArrayInit(2, sizeof(void*));
taosArrayPush(arrs, &tr->total);
taosArrayPush(arrs, &tr->added);
iUnion(arrs, result);
taosArrayDestroy(arrs);
iExcept(result, tr->deled);
}
static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdxTempResult* tr) {
int32_t sz = taosArrayGetSize(result);
if (sz > 0) {
// TODO(yihao): remove duplicate tableid
TFileValue* lv = taosArrayGetP(result, sz - 1);
// indexError("merge colVal: %s", lv->colVal);
if (strcmp(lv->colVal, tv->colVal) == 0) {
taosArrayAddAll(lv->tableId, tv->tableId);
tfileValueDestroy(tv);
if (tfv != NULL && strcmp(lv->colVal, tfv->colVal) != 0) {
sIdxTempResultMergeTo(lv->tableId, tr);
sIdxTempResultClear(tr);
taosArrayPush(result, &tfv);
} else if (tfv == NULL) {
// handle last iterator
sIdxTempResultMergeTo(lv->tableId, tr);
} else {
taosArrayPush(result, &tv);
// temp result saved in help
tfileValueDestroy(tfv);
}
} else {
taosArrayPush(result, &tv);
taosArrayPush(result, &tfv);
}
}
static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv) {
// opt
char* colVal = (cv != NULL) ? cv->colVal : tv->colVal;
// design merge-algorithm later, too complicated to handle all kind of situation
static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTempResult* tr) {
char* colVal = (cv != NULL) ? cv->colVal : tv->colVal;
TFileValue* tfv = tfileValueCreate(colVal);
indexMayMergeTempToFinalResult(result, tfv, tr);
if (cv != NULL) {
uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0);
if (cv->type == ADD_VALUE) {
taosArrayAddAll(tfv->tableId, cv->val);
INDEX_MERGE_ADD_DEL(tr->deled, tr->added, id)
} else if (cv->type == DEL_VALUE) {
} else if (cv->type == UPDATE_VALUE) {
} else {
// do nothing
INDEX_MERGE_ADD_DEL(tr->added, tr->deled, id)
}
}
if (tv != NULL) {
// opt later
taosArrayAddAll(tfv->tableId, tv->val);
taosArrayAddAll(tr->total, tv->val);
}
indexMergeSameKey(result, tfv);
}
static void indexDestroyTempResult(SArray* result) {
static void indexDestroyFinalResult(SArray* result) {
int32_t sz = result ? taosArrayGetSize(result) : 0;
for (size_t i = 0; i < sz; i++) {
TFileValue* tv = taosArrayGetP(result, i);
@ -443,6 +496,7 @@ static void indexDestroyTempResult(SArray* result) {
}
taosArrayDestroy(result);
}
int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
if (sIdx == NULL) {
return -1;
@ -467,6 +521,8 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
bool cn = cacheIter ? cacheIter->next(cacheIter) : false;
bool tn = tfileIter ? tfileIter->next(tfileIter) : false;
SIdxTempResult* tr = sIdxTempResultCreate();
while (cn == true || tn == true) {
IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL;
IterateValue* tv = (tn == true) ? tfileIter->getValue(tfileIter) : NULL;
@ -480,19 +536,22 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
comp = 1;
}
if (comp == 0) {
indexMergeCacheAndTFile(result, cv, tv);
indexMergeCacheAndTFile(result, cv, tv, tr);
cn = cacheIter->next(cacheIter);
tn = tfileIter->next(tfileIter);
} else if (comp < 0) {
indexMergeCacheAndTFile(result, cv, NULL);
indexMergeCacheAndTFile(result, cv, NULL, tr);
cn = cacheIter->next(cacheIter);
} else {
indexMergeCacheAndTFile(result, NULL, tv);
indexMergeCacheAndTFile(result, NULL, tv, tr);
tn = tfileIter->next(tfileIter);
}
}
indexMayMergeTempToFinalResult(result, NULL, tr);
sIdxTempResultDestroy(tr);
int ret = indexGenTFile(sIdx, pCache, result);
indexDestroyTempResult(result);
indexDestroyFinalResult(result);
indexCacheDestroyImm(pCache);

13
source/libs/index/src/index_cache.c
View File

@ -267,6 +267,10 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA
SSkipListNode* node = tSkipListIterGet(iter);
if (node != NULL) {
CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
// if (c->operaType == ADD_VALUE) {
//} else if (c->operaType == DEL_VALUE) {
//}
if (c->operaType == ADD_VALUE || qtype == QUERY_TERM) {
if (strcmp(c->colVal, ct->colVal) == 0) {
taosArrayPush(result, &c->uid);
@ -411,17 +415,8 @@ static bool indexCacheIteratorNext(Iterate* itera) {
SSkipListNode* node = tSkipListIterGet(iter);
CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node);
// equal func
// if (iv->colVal != NULL && ct->colVal != NULL) {
// if (0 == strcmp(iv->colVal, ct->colVal)) { if (iv->type == ADD_VALUE) }
//} else {
// tIterVal.colVal = calloc(1, strlen(ct->colVal) + 1);
// tIterval.colVal = tstrdup(ct->colVal);
//}
iv->type = ct->operaType;
iv->colVal = tstrdup(ct->colVal);
// iv->colVal = calloc(1, strlen(ct->colVal) + 1);
// memcpy(iv->colVal, ct->colVal, strlen(ct->colVal));
taosArrayPush(iv->val, &ct->uid);
}

18
source/libs/index/src/index_fst_counting_writer.c
View File

@ -63,9 +63,9 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off
}
static int writeCtxGetSize(WriterCtx* ctx) {
if (ctx->type == TFile) {
struct stat fstat;
stat(ctx->file.buf, &fstat);
return fstat.st_size;
int64_t file_size = 0;
taosStatFile(ctx->file.buf, &file_size, NULL);
return (int)file_size;
}
return 0;
}
@ -99,9 +99,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
// ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
struct stat fstat;
stat(path, &fstat);
ctx->file.size = fstat.st_size;
int64_t file_size = 0;
taosFStatFile(ctx->file.pFile, &file_size, NULL);
ctx->file.size = (int)file_size;
#ifdef USE_MMAP
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
#endif
@ -142,8 +142,10 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
#endif
}
if (ctx->file.readOnly == false) {
struct stat fstat;
stat(ctx->file.buf, &fstat);
int64_t file_size = 0;
taosStatFile(ctx->file.buf, &file_size, NULL);
// struct stat fstat;
// stat(ctx->file.buf, &fstat);
// indexError("write file size: %d", (int)(fstat.st_size));
}
if (remove) { unlink(ctx->file.buf); }

2
source/libs/index/src/index_tfile.c
View File

@ -102,7 +102,6 @@ void tfileCacheDestroy(TFileCache* tcache) {
if (tcache == NULL) {
return;
}
// free table cache
TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
while (reader) {
@ -429,6 +428,7 @@ static bool tfileIteratorNext(Iterate* iiter) {
return false;
}
iv->type = ADD_VALUE; // value in tfile always ADD_VALUE
iv->colVal = colVal;
return true;
// std::string key(ch, sz);

23
source/libs/index/src/index_util.c
View File

@ -14,6 +14,7 @@
*/
#include "index_util.h"
#include "index.h"
typedef struct MergeIndex {
int idx;
int len;
@ -111,6 +112,26 @@ void iUnion(SArray *inters, SArray *final) {
break;
}
}
tfree(mi);
}
void iExcept(SArray *total, SArray *except) {
int32_t tsz = taosArrayGetSize(total);
int32_t esz = taosArrayGetSize(except);
if (esz == 0 || tsz == 0) {
return;
}
int vIdx = 0;
for (int i = 0; i < tsz; i++) {
uint64_t val = *(uint64_t *)taosArrayGet(total, i);
int idx = iBinarySearch(except, 0, esz - 1, val);
if (idx >= 0 && idx < esz && *(uint64_t *)taosArrayGet(except, idx) == val) {
continue;
}
taosArraySet(total, vIdx, &val);
vIdx += 1;
}
taosArrayPopTailBatch(total, tsz - vIdx);
}

2
source/libs/index/test/fstTest.cc
View File

@ -19,7 +19,7 @@ static std::string fileName = "/tmp/tindex.tindex";
class FstWriter {
public:
FstWriter() {
remove(fileName.c_str());
taosRemoveFile(fileName.c_str());
_wc = writerCtxCreate(TFile, fileName.c_str(), false, 64 * 1024 * 1024);
_b = fstBuilderCreate(_wc, 0);
}

16
source/libs/index/test/jsonUT.cc
View File

@ -105,6 +105,22 @@ TEST_F(JsonEnv, testWriteMillonData) {
}
indexMultiTermDestroy(terms);
}
{
std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx");
for (uint i = 0; i < 1000; i++) {
colVal[i % colVal.size()] = '0' + i % 128;
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
colVal.c_str(), colVal.size());
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 1000; i++) {
tIndexJsonPut(index, terms, i);
}
indexMultiTermDestroy(terms);
}
}
{
std::string colName("voltagefdadfa");
std::string colVal("abxxxxxxxxxxxx");

81
source/libs/index/test/utilUT.cc
View File

@ -224,3 +224,84 @@ TEST_F(UtilEnv, 04union) {
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 12);
}
TEST_F(UtilEnv, 01Except) {
SArray *total = taosArrayInit(4, sizeof(uint64_t));
{
uint64_t arr1[] = {1, 4, 5, 6};
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(total, &arr1[i]);
}
}
SArray *except = taosArrayInit(4, sizeof(uint64_t));
{
uint64_t arr1[] = {1, 4, 5, 6};
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(except, &arr1[i]);
}
}
iExcept(total, except);
ASSERT_EQ(taosArrayGetSize(total), 0);
taosArrayClear(total);
taosArrayClear(except);
{
uint64_t arr1[] = {1, 4, 5, 6, 7, 8};
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(total, &arr1[i]);
}
}
{
uint64_t arr1[] = {2, 4, 5, 6};
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(except, &arr1[i]);
}
}
iExcept(total, except);
ASSERT_EQ(taosArrayGetSize(total), 3);
taosArrayClear(total);
taosArrayClear(except);
{
uint64_t arr1[] = {1, 4, 5, 6, 7, 8, 10, 100};
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(total, &arr1[i]);
}
}
{
uint64_t arr1[] = {2, 4, 5, 6};
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(except, &arr1[i]);
}
}
iExcept(total, except);
ASSERT_EQ(taosArrayGetSize(total), 5);
ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1);
ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 7);
ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 2), 8);
ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 3), 10);
ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 4), 100);
taosArrayClear(total);
taosArrayClear(except);
{
uint64_t arr1[] = {1, 100};
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(total, &arr1[i]);
}
}
{
uint64_t arr1[] = {2, 4, 5, 6};
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(except, &arr1[i]);
}
}
iExcept(total, except);
ASSERT_EQ(taosArrayGetSize(total), 2);
ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1);
ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 100);
}

18
source/libs/monitor/inc/monInt.h
View File

@ -23,18 +23,24 @@
#include "tjson.h"
typedef struct {
SRWLatch lock;
SArray *logs; // array of SMonLogItem
int32_t maxLogs;
const char *server;
uint16_t port;
} SMonitor;
int64_t ts;
ELogLevel level;
char content[MON_LOG_LEN];
} SMonLogItem;
typedef struct SMonInfo {
SArray *logs; // array of SMonLogItem
SJson *pJson;
} SMonInfo;
typedef struct {
pthread_mutex_t lock;
SArray *logs; // array of SMonLogItem
int32_t maxLogs;
const char *server;
uint16_t port;
} SMonitor;
#ifdef __cplusplus
}
#endif

62
source/libs/monitor/src/monitor.c
View File

@ -22,6 +22,19 @@
static SMonitor tsMonitor = {0};
void monRecordLog(int64_t ts, ELogLevel level, const char *content) {
pthread_mutex_lock(&tsMonitor.lock);
int32_t size = taosArrayGetSize(tsMonitor.logs);
if (size < tsMonitor.maxLogs) {
SMonLogItem item = {.ts = ts, .level = level};
SMonLogItem *pItem = taosArrayPush(tsMonitor.logs, &item);
if (pItem != NULL) {
tstrncpy(pItem->content, content, MON_LOG_LEN);
}
}
pthread_mutex_unlock(&tsMonitor.lock);
}
int32_t monInit(const SMonCfg *pCfg) {
tsMonitor.logs = taosArrayInit(16, sizeof(SMonLogItem));
if (tsMonitor.logs == NULL) {
@ -32,24 +45,16 @@ int32_t monInit(const SMonCfg *pCfg) {
tsMonitor.maxLogs = pCfg->maxLogs;
tsMonitor.server = pCfg->server;
tsMonitor.port = pCfg->port;
taosInitRWLatch(&tsMonitor.lock);
tsLogFp = monRecordLog;
pthread_mutex_init(&tsMonitor.lock, NULL);
return 0;
}
void monCleanup() {
tsLogFp = NULL;
taosArrayDestroy(tsMonitor.logs);
tsMonitor.logs = NULL;
}
void monAddLogItem(SMonLogItem *pItem) {
taosWLockLatch(&tsMonitor.lock);
int32_t size = taosArrayGetSize(tsMonitor.logs);
if (size >= tsMonitor.maxLogs) {
uInfo("too many logs for monitor");
} else {
taosArrayPush(tsMonitor.logs, pItem);
}
taosWUnLockLatch(&tsMonitor.lock);
pthread_mutex_destroy(&tsMonitor.lock);
}
SMonInfo *monCreateMonitorInfo() {
@ -59,10 +64,10 @@ SMonInfo *monCreateMonitorInfo() {
return NULL;
}
taosWLockLatch(&tsMonitor.lock);
pthread_mutex_lock(&tsMonitor.lock);
pMonitor->logs = taosArrayDup(tsMonitor.logs);
taosArrayClear(tsMonitor.logs);
taosWUnLockLatch(&tsMonitor.lock);
pthread_mutex_unlock(&tsMonitor.lock);
pMonitor->pJson = tjsonCreateObject();
if (pMonitor->pJson == NULL || pMonitor->logs == NULL) {
@ -128,11 +133,11 @@ void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) {
SJson *pMnodesJson = tjsonAddArrayToObject(pJson, "mnodes");
if (pMnodesJson == NULL) return;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) {
for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) {
SJson *pMnodeJson = tjsonCreateObject();
if (pMnodeJson == NULL) continue;
SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->dnodes, i);
SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i);
tjsonAddDoubleToObject(pMnodeJson, "mnode_id", pMnodeDesc->mnode_id);
tjsonAddStringToObject(pMnodeJson, "mnode_ep", pMnodeDesc->mnode_ep);
tjsonAddStringToObject(pMnodeJson, "role", pMnodeDesc->role);
@ -270,6 +275,21 @@ void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) {
tjsonAddDoubleToObject(pTempdirJson, "total", pInfo->tempdir.size.total);
}
static const char *monLogLevelStr(ELogLevel level) {
switch (level) {
case DEBUG_ERROR:
return "error";
case DEBUG_INFO:
return "info";
case DEBUG_DEBUG:
return "debug";
case DEBUG_TRACE:
return "trace";
default:
return "undefine";
}
}
static void monSetLogInfo(SMonInfo *pMonitor) {
SJson *pJson = tjsonCreateObject();
if (pJson == NULL) return;
@ -291,7 +311,7 @@ static void monSetLogInfo(SMonInfo *pMonitor) {
taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI);
tjsonAddStringToObject(pLogJson, "ts", buf);
tjsonAddDoubleToObject(pLogJson, "level", pLogItem->level);
tjsonAddStringToObject(pLogJson, "level", monLogLevelStr(pLogItem->level));
tjsonAddStringToObject(pLogJson, "content", pLogItem->content);
if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson);
@ -303,25 +323,25 @@ static void monSetLogInfo(SMonInfo *pMonitor) {
SJson *pLogError = tjsonCreateObject();
if (pLogError == NULL) return;
tjsonAddStringToObject(pLogError, "level", "error");
tjsonAddDoubleToObject(pLogError, "total", 1);
tjsonAddDoubleToObject(pLogError, "total", tsNumOfErrorLogs);
if (tjsonAddItemToArray(pSummaryJson, pLogError) != 0) tjsonDelete(pLogError);
SJson *pLogInfo = tjsonCreateObject();
if (pLogInfo == NULL) return;
tjsonAddStringToObject(pLogInfo, "level", "info");
tjsonAddDoubleToObject(pLogInfo, "total", 1);
tjsonAddDoubleToObject(pLogInfo, "total", tsNumOfInfoLogs);
if (tjsonAddItemToArray(pSummaryJson, pLogInfo) != 0) tjsonDelete(pLogInfo);
SJson *pLogDebug = tjsonCreateObject();
if (pLogDebug == NULL) return;
tjsonAddStringToObject(pLogDebug, "level", "debug");
tjsonAddDoubleToObject(pLogDebug, "total", 1);
tjsonAddDoubleToObject(pLogDebug, "total", tsNumOfDebugLogs);
if (tjsonAddItemToArray(pSummaryJson, pLogDebug) != 0) tjsonDelete(pLogDebug);
SJson *pLogTrace = tjsonCreateObject();
if (pLogTrace == NULL) return;
tjsonAddStringToObject(pLogTrace, "level", "trace");
tjsonAddDoubleToObject(pLogTrace, "total", 1);
tjsonAddDoubleToObject(pLogTrace, "total", tsNumOfTraceLogs);
if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace);
}

33
source/libs/monitor/test/monTest.cpp
View File

@ -193,37 +193,14 @@ void MonitorTest::GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) {
}
void MonitorTest::AddLogInfo1() {
SMonLogItem log1 = {0};
log1.ts = taosGetTimestampMs();
log1.level = 1;
strcpy(log1.content, "1 -------------------------- a");
monAddLogItem(&log1);
SMonLogItem log2 = {0};
log2.ts = taosGetTimestampMs();
log2.level = 1;
strcpy(log2.content, "1 ------------------------ b");
monAddLogItem(&log2);
SMonLogItem log3 = {0};
log3.ts = taosGetTimestampMs();
log3.level = 1;
strcpy(log3.content, "1 ------- c");
monAddLogItem(&log3);
monRecordLog(taosGetTimestampMs(), DEBUG_INFO, "1 -------------------------- a");
monRecordLog(taosGetTimestampMs(), DEBUG_ERROR, "1 ------------------------ b");
monRecordLog(taosGetTimestampMs(), DEBUG_DEBUG, "1 ------- c");
}
void MonitorTest::AddLogInfo2() {
SMonLogItem log1;
log1.ts = taosGetTimestampMs();
log1.level = 01;
strcpy(log1.content, "2 ------- a");
monAddLogItem(&log1);
SMonLogItem log2;
log2.ts = taosGetTimestampMs();
log2.level = 0;
strcpy(log2.content, "2 ------- b");
monAddLogItem(&log2);
monRecordLog(taosGetTimestampMs(), DEBUG_ERROR, "2 ------- a");
monRecordLog(taosGetTimestampMs(), DEBUG_ERROR, "2 ------- b");
}
TEST_F(MonitorTest, 01_Full) {

2
source/libs/parser/src/insertParser.c
View File

@ -153,7 +153,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) {
if (NULL == dst) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
taosHashGetDup(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
dst->numOfTables = src->numOfTables;
dst->size = src->size;
TSWAP(dst->pData, src->pData, char*);

6
source/libs/parser/test/mockCatalog.cpp
View File

@ -13,19 +13,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mockCatalog.h"
#include <iostream>
#include "stub.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat"
#include "addr_any.h"
#include <addr_any.h>
#pragma GCC diagnostic pop
#include "mockCatalog.h"
namespace {
void generateTestT1(MockCatalogService* mcs) {

2
source/libs/parser/test/mockCatalogService.cpp
View File

@ -15,10 +15,10 @@
#include "mockCatalogService.h"
#include "tep.h"
#include <iomanip>
#include <iostream>
#include <map>
#include "tdatablock.h"
#include "tname.h"
#include "ttypes.h"

4
source/libs/parser/test/parserTests.cpp
View File

@ -13,10 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <function.h>
#include <gtest/gtest.h>
#include <iostream>
#include "tglobal.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
@ -25,6 +23,8 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "function.h"
#include "tglobal.h"
#include "astGenerator.h"
#include "parserInt.h"
#include "taos.h"

2
source/libs/parser/test/plannerTest.cpp
View File

@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <function.h>
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
@ -25,6 +24,7 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "function.h"
#include "astGenerator.h"
#include "parserInt.h"
#include "taos.h"

66
source/libs/qcom/src/queryUtil.c
View File

@ -1,23 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tmsg.h"
#include "query.h"
#include "tglobal.h"
#include "tsched.h"
#include "tmsg.h"
#include "trpc.h"
#include "tsched.h"
#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS)
#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS)
static struct SSchema _s = {
.colId = TSDB_TBNAME_COLUMN_INDEX,
.type = TSDB_DATA_TYPE_BINARY,
.type = TSDB_DATA_TYPE_BINARY,
.bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE,
.name = "tbname",
};
const SSchema* tGetTbnameColumnSchema() {
return &_s;
}
const SSchema* tGetTbnameColumnSchema() { return &_s; }
static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) {
int32_t rowLen = 0;
@ -87,7 +100,7 @@ int32_t initTaskQueue() {
double factor = 4.0;
int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2);
int32_t queueSize = tsMaxConnections * 2;
pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc");
if (NULL == pTaskQueue) {
@ -96,19 +109,21 @@ int32_t initTaskQueue() {
}
qDebug("task queue is initialized, numOfThreads: %d", numOfThreads);
return 0;
}
int32_t cleanupTaskQueue() {
taosCleanUpScheduler(pTaskQueue);
return 0;
}
static void execHelper(struct SSchedMsg* pSchedMsg) {
assert(pSchedMsg != NULL && pSchedMsg->ahandle != NULL);
__async_exec_fn_t execFn = (__async_exec_fn_t) pSchedMsg->ahandle;
int32_t code = execFn(pSchedMsg->thandle);
__async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle;
int32_t code = execFn(pSchedMsg->thandle);
if (code != 0 && pSchedMsg->msg != NULL) {
*(int32_t*) pSchedMsg->msg = code;
*(int32_t*)pSchedMsg->msg = code;
}
}
@ -116,34 +131,33 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code)
assert(execFn != NULL);
SSchedMsg schedMsg = {0};
schedMsg.fp = execHelper;
schedMsg.fp = execHelper;
schedMsg.ahandle = execFn;
schedMsg.thandle = execParam;
schedMsg.msg = code;
schedMsg.msg = code;
taosScheduleTask(pTaskQueue, &schedMsg);
return 0;
}
int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) {
char *pMsg = rpcMallocCont(pInfo->msgInfo.len);
int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) {
char* pMsg = rpcMallocCont(pInfo->msgInfo.len);
if (NULL == pMsg) {
qError("0x%"PRIx64" msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType));
qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType));
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return terrno;
}
memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len);
SRpcMsg rpcMsg = {
.msgType = pInfo->msgType,
.pCont = pMsg,
.contLen = pInfo->msgInfo.len,
.ahandle = (void*) pInfo,
.handle = pInfo->msgInfo.handle,
.code = 0
};
SRpcMsg rpcMsg = {.msgType = pInfo->msgType,
.pCont = pMsg,
.contLen = pInfo->msgInfo.len,
.ahandle = (void*)pInfo,
.handle = pInfo->msgInfo.handle,
.code = 0};
assert(pInfo->fp != NULL);
rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId);
return TSDB_CODE_SUCCESS;
}
}

6
source/libs/qworker/test/qworkerTests.cpp
View File

@ -14,7 +14,6 @@
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic push
@ -26,18 +25,19 @@
#pragma GCC diagnostic ignored "-Wformat"
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
#pragma GCC diagnostic ignored "-Wpointer-arith"
#include <addr_any.h>
#include "os.h"
#include "tglobal.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "tep.h"
#include "tdatablock.h"
#include "trpc.h"
#include "planner.h"
#include "qworker.h"
#include "stub.h"
#include "addr_any.h"
#include "executor.h"
#include "dataSinkMgt.h"

10
source/libs/scalar/inc/filterInt.h
View File

@ -20,13 +20,13 @@
extern "C" {
#endif
#include "query.h"
#include "querynodes.h"
#include "scalar.h"
#include "tcommon.h"
#include "tdatablock.h"
#include "thash.h"
#include "tname.h"
#include "tcommon.h"
#include "scalar.h"
#include "querynodes.h"
#include "query.h"
#include "tep.h"
#define FILTER_DEFAULT_GROUP_SIZE 4
#define FILTER_DEFAULT_UNIT_SIZE 4

6
source/libs/scalar/src/filter.c
View File

@ -16,11 +16,11 @@
#include <tlog.h>
#include "thash.h"
//#include "queryLog.h"
#include "tcompare.h"
#include "filter.h"
#include "filterInt.h"
#include "sclInt.h"
#include "filter.h"
#include "tep.h"
#include "tcompare.h"
#include "tdatablock.h"
OptrStr gOptrStr[] = {
{0, "invalid"},

10
source/libs/scalar/src/scalar.c
View File

@ -1,11 +1,11 @@
#include "nodes.h"
#include "tcommon.h"
#include "querynodes.h"
#include "function.h"
#include "functionMgt.h"
#include "sclvector.h"
#include "nodes.h"
#include "querynodes.h"
#include "sclInt.h"
#include "tep.h"
#include "sclvector.h"
#include "tcommon.h"
#include "tdatablock.h"
int32_t scalarGetOperatorParamNum(EOperatorType type) {
if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type

12
source/libs/scalar/src/sclvector.c
View File

@ -15,15 +15,15 @@
#include "os.h"
#include "ttypes.h"
#include "sclvector.h"
#include "tcompare.h"
#include "querynodes.h"
#include "filter.h"
#include "filterInt.h"
#include "query.h"
#include "querynodes.h"
#include "sclInt.h"
#include "tep.h"
#include "filter.h"
#include "sclvector.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "ttypes.h"
//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i]));

Some files were not shown because too many files have changed in this diff Show More