diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt index d4d5a6365f..e0f6ae64bd 100644 --- a/contrib/test/craft/CMakeLists.txt +++ b/contrib/test/craft/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable(simulate_vnode "simulate_vnode.c") -target_link_libraries(simulate_vnode craft lz4 uv_a) \ No newline at end of file +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file diff --git a/contrib/test/traft/single_node/CMakeLists.txt b/contrib/test/traft/single_node/CMakeLists.txt index 84b65978b9..666ce271b8 100644 --- a/contrib/test/traft/single_node/CMakeLists.txt +++ b/contrib/test/traft/single_node/CMakeLists.txt @@ -3,4 +3,4 @@ target_sources(singleNode PRIVATE "singleNode.c" ) -target_link_libraries(singleNode traft lz4 uv_a) +target_link_libraries(singleNode PUBLIC traft lz4 uv_a) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a04e2afc94..d0ec5c9296 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -25,24 +25,6 @@ extern "C" { #endif -// typedef struct STimeWindow { -// TSKEY skey; -// TSKEY ekey; -// } STimeWindow; - -// typedef struct { -// int32_t dataLen; -// char name[TSDB_TABLE_FNAME_LEN]; -// char *data; -// } STagData; - -// typedef struct SSchema { -// uint8_t type; -// char name[TSDB_COL_NAME_LEN]; -// int16_t colId; -// int16_t bytes; -// } SSchema; - enum { TMQ_CONF__RESET_OFFSET__LATEST = -1, TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, @@ -50,7 +32,8 @@ enum { }; enum { - TMQ_MSG_TYPE__POLL_RSP = 0, + TMQ_MSG_TYPE__DUMMY = 0, + TMQ_MSG_TYPE__POLL_RSP, TMQ_MSG_TYPE__EP_RSP, }; @@ -285,4 +268,4 @@ typedef struct SSessionWindow { } #endif -#endif /*_TD_COMMON_DEF_H_*/ +#endif /*_TD_COMMON_DEF_H_*/ diff --git a/include/common/tep.h b/include/common/tdatablock.h similarity index 100% rename from include/common/tep.h rename to include/common/tdatablock.h diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 25d668e788..1b08d6f241 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -70,7 +70,7 @@ typedef uint16_t tmsg_t; typedef enum { HEARTBEAT_TYPE_MQ = 0, - HEARTBEAT_TYPE_QUERY = 1, + HEARTBEAT_TYPE_QUERY, // types can be added here // HEARTBEAT_TYPE_MAX diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index a4c0be27ab..d4af51fc21 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -89,25 +89,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds); */ int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext); -/** - * - * Retrieve the actual results to fill the response message payload. - * Note that this function must be executed after qRetrieveQueryResultInfo is invoked. - * - * @param tinfo tinfo object - * @param pRsp response message - * @param contLen payload length - * @return - */ -//int32_t qDumpRetrieveResult(qTaskInfo_t tinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); - -/** - * return the transporter context (RPC) - * @param tinfo - * @return - */ -void* qGetResultRetrieveMsg(qTaskInfo_t tinfo); - /** * kill the ongoing query and free the query handle and corresponding resources automatically * @param tinfo qhandle diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index b3b6091b95..1695edd983 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -18,6 +18,7 @@ #include "tarray.h" #include "tdef.h" +#include "tlog.h" #ifdef __cplusplus extern "C" { @@ -85,21 +86,21 @@ typedef struct { typedef struct { float uptime; // day - float cpu_engine; - float cpu_system; + double cpu_engine; + double cpu_system; float cpu_cores; - float mem_engine; // MB - float mem_system; // MB - float mem_total; // MB - float disk_engine; // GB - float disk_used; // GB - float disk_total; // GB - float net_in; // Kb/s - float net_out; // Kb/s - float io_read; // Mb/s - float io_write; // Mb/s - float io_read_disk; // Mb/s - float io_write_disk; // Mb/s + int64_t mem_engine; // KB + int64_t mem_system; // KB + int64_t mem_total; // KB + int64_t disk_engine; // Byte + int64_t disk_used; // Byte + int64_t disk_total; // Byte + double net_in; // bytes per second + double net_out; // bytes per second + double io_read; + double io_write; + double io_read_disk; + double io_write_disk; int32_t req_select; float req_select_rate; int32_t req_insert; @@ -116,7 +117,7 @@ typedef struct { typedef struct { char name[TSDB_FILENAME_LEN]; - int32_t level; + int8_t level; SDiskSize size; } SMonDiskDesc; @@ -126,12 +127,6 @@ typedef struct { SMonDiskDesc tempdir; } SMonDiskInfo; -typedef struct { - int64_t ts; - int8_t level; - char content[MON_LOG_LEN]; -} SMonLogItem; - typedef struct SMonInfo SMonInfo; typedef struct { @@ -142,7 +137,7 @@ typedef struct { int32_t monInit(const SMonCfg *pCfg); void monCleanup(); -void monAddLogItem(SMonLogItem *pItem); +void monRecordLog(int64_t ts, ELogLevel level, const char *content); SMonInfo *monCreateMonitorInfo(); void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 6d3f97fc4e..1f56254476 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -168,20 +168,18 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); - #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE #define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE -#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) - +#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); }} while(0) +#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); }} while(0) +#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); }} while(0) +#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0) +#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index fedb487931..a93180800e 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -19,10 +19,12 @@ extern "C" { #endif +#include "tcommon.h" +#include "nodes.h" + typedef struct SFilterInfo SFilterInfo; typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); - enum { FLT_OPTION_NO_REWRITE = 1, FLT_OPTION_TIMESTAMP = 2, @@ -34,7 +36,6 @@ typedef struct SFilterColumnParam{ SArray* pDataBlock; } SFilterColumnParam; - extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index d6fdadc1fb..fddf18c571 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -21,7 +21,7 @@ extern "C" { #endif #include -#include +#include #include "taosdef.h" #include "trpc.h" diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index b6d10c81bf..1b41da33bb 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -17,6 +17,7 @@ #define _TD_TFS_H_ #include "tdef.h" +#include "monitor.h" #ifdef __cplusplus extern "C" { @@ -237,6 +238,14 @@ const STfsFile *tfsReaddir(STfsDir *pDir); */ void tfsClosedir(STfsDir *pDir); +/** + * @brief Get disk info of tfs. + * + * @param pTfs The fs object. + * @param pInfo The info object. + */ +int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo); + #ifdef __cplusplus } #endif diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index f90dbb97fe..32fcd95948 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -24,43 +24,41 @@ extern "C" { #endif -extern int32_t wDebugFlag; - -#define wFatal(...) \ - { \ - if (wDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \ - } \ +#define wFatal(...) \ + { \ + if (wDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } -#define wError(...) \ - { \ - if (wDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \ - } \ +#define wError(...) \ + { \ + if (wDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } -#define wWarn(...) \ - { \ - if (wDebugFlag & DEBUG_WARN) { \ - taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \ - } \ +#define wWarn(...) \ + { \ + if (wDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } -#define wInfo(...) \ - { \ - if (wDebugFlag & DEBUG_INFO) { \ - taosPrintLog("WAL ", 255, __VA_ARGS__); \ - } \ +#define wInfo(...) \ + { \ + if (wDebugFlag & DEBUG_INFO) { \ + taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } -#define wDebug(...) \ - { \ - if (wDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ - } \ +#define wDebug(...) \ + { \ + if (wDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \ + } \ } -#define wTrace(...) \ - { \ - if (wDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ - } \ +#define wTrace(...) \ + { \ + if (wDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \ + } \ } #define WAL_HEAD_VER 0 diff --git a/include/os/os.h b/include/os/os.h index f020af5a65..a58e798d38 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -44,6 +44,7 @@ extern "C" { #include #include #include +#include #include #include #include diff --git a/include/os/osDir.h b/include/os/osDir.h index eda83ea0ea..223c603352 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -21,7 +21,7 @@ extern "C" { #endif void taosRemoveDir(const char *dirname); -int32_t taosDirExist(char *dirname); +bool taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 4ac073f6c2..1426ba87f6 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -28,11 +28,11 @@ extern char tsCharset[]; extern char tsLocale[]; extern int8_t tsDaylight; extern bool tsEnableCoreFile; -extern int64_t tsPageSize; +extern int64_t tsPageSizeKB; extern int64_t tsOpenMax; extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; -extern int32_t tsTotalMemoryMB; +extern float tsNumOfCores; +extern int64_t tsTotalMemoryKB; extern char configDir[]; extern char tsDataDir[]; diff --git a/include/os/osFile.h b/include/os/osFile.h index cbd5d693c5..6ddf1e33c6 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -22,11 +22,18 @@ extern "C" { #include "osSocket.h" +// If the error is in a third-party library, place this header file under the third-party library header file. #ifndef ALLOW_FORBID_FUNC #define open OPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID - // #define close CLOSE_FUNC_TAOS_FORBID - // #define fclose FCLOSE_FUNC_TAOS_FORBID + #define access ACCESS_FUNC_TAOS_FORBID + #define stat STAT_FUNC_TAOS_FORBID + #define lstat LSTAT_FUNC_TAOS_FORBID + #define fstat FSTAT_FUNC_TAOS_FORBID + #define close CLOSE_FUNC_TAOS_FORBID + #define fclose FCLOSE_FUNC_TAOS_FORBID + #define fsync FSYNC_FUNC_TAOS_FORBID + // #define fflush FFLUSH_FUNC_TAOS_FORBID #endif #ifndef PATH_MAX @@ -43,7 +50,13 @@ typedef struct TdFile *TdFilePtr; #define TD_FILE_TEXT 0x0020 #define TD_FILE_AUTO_DEL 0x0040 #define TD_FILE_EXCL 0x0080 -#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile +#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); + +#define TD_FILE_ACCESS_EXIST_OK 0x1 +#define TD_FILE_ACCESS_READ_OK 0x2 +#define TD_FILE_ACCESS_WRITE_OK 0x4 +bool taosCheckAccessFile(const char *pathname, int mode); int32_t taosLockFile(TdFilePtr pFile); int32_t taosUnLockFile(TdFilePtr pFile); @@ -51,9 +64,9 @@ int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUmaskFile(int32_t maskVal); int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); +int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno); int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); - -TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); +bool taosCheckExistFile(const char *pathname); int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length); @@ -62,7 +75,7 @@ int32_t taosFsyncFile(TdFilePtr pFile); int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); -void taosFprintfFile(TdFilePtr pFile, const char *format, ...); +void taosFprintfFile(TdFilePtr pFile, const char *format, ...); int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); int32_t taosEOFFile(TdFilePtr pFile); @@ -70,8 +83,9 @@ int64_t taosCloseFile(TdFilePtr *ppFile); int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); +int32_t taosRemoveFile(const char *path); -void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); +void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size); int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); @@ -79,7 +93,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); bool taosValidFile(TdFilePtr pFile); -int taosGetErrorFile(TdFilePtr pFile); +int32_t taosGetErrorFile(TdFilePtr pFile); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index bf9d7662e6..9081fa9715 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -34,19 +34,19 @@ typedef struct { } SDiskSpace; void taosGetSystemInfo(); -bool taosGetEmail(char *email, int32_t maxLen); -bool taosGetOsReleaseName(char *releaseName, int32_t maxLen); -bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores); -int32_t taosGetCpuCores(); -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); -bool taosGetTotalSysMemoryKB(uint64_t *kb); -bool taosGetProcMemory(float *memoryUsedMB); // -bool taosGetSysMemory(float *memoryUsedMB); // +int32_t taosGetEmail(char *email, int32_t maxLen); +int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); +int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores); +int32_t taosGetCpuCores(float *numOfCores); +int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine); +int32_t taosGetTotalMemory(int64_t *totalKB); +int32_t taosGetProcMemory(int64_t *usedKB); +int32_t taosGetSysMemory(int64_t *usedKB); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); -bool taosReadProcIO(int64_t *rchars, int64_t *wchars); -bool taosGetProcIO(float *readKB, float *writeKB); -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes); -bool taosGetBandSpeed(float *bandSpeedKb); +int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); +int32_t taosGetIOSpeed(double *readKB, double *writeKB, double *readDiskKB, double *writeDiskKB); +int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes); +int32_t taosGetBandSpeed(double *receive_bytes_per_sec, double *transmit_bytes_per_sec); int32_t taosSystem(const char *cmd); void taosKillSystem(); diff --git a/include/util/tcache.h b/include/util/tcache.h index 0de3ab3a28..b5c1578380 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -40,55 +40,9 @@ typedef struct SCacheStatis { int64_t refreshCount; } SCacheStatis; -struct STrashElem; - -typedef struct SCacheDataNode { - uint64_t addedTime; // the added time when this element is added or updated into cache - uint64_t lifespan; // life duration when this element should be remove from cache - uint64_t expireTime; // expire time - uint64_t signature; - struct STrashElem *pTNodeHeader; // point to trash node head - uint16_t keySize : 15; // max key size: 32kb - bool inTrashcan : 1; // denote if it is in trash or not - uint32_t size; // allocated size for current SCacheDataNode - T_REF_DECLARE() - char *key; - char data[]; -} SCacheDataNode; - -typedef struct STrashElem { - struct STrashElem *prev; - struct STrashElem *next; - SCacheDataNode *pData; -} STrashElem; - -/* - * to accommodate the old data which has the same key value of new one in hashList - * when an new node is put into cache, if an existed one with the same key: - * 1. if the old one does not be referenced, update it. - * 2. otherwise, move the old one to pTrash, addedTime the new one. - * - * when the node in pTrash does not be referenced, it will be release at the expired expiredTime - */ -typedef struct { - int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included. - int64_t refreshTime; - STrashElem *pTrash; - char *name; - SCacheStatis statistics; - SHashObj *pHashTable; - __cache_free_fn_t freeFp; - uint32_t numOfElemsInTrash; // number of element in trash - uint8_t deleting; // set the deleting flag to stop refreshing ASAP. - pthread_t refreshWorker; - bool extendLifespan; // auto extend life span when one item is accessed. - int64_t checkTick; // tick used to record the check times of the refresh threads -#if defined(LINUX) - pthread_rwlock_t lock; -#else - pthread_mutex_t lock; -#endif -} SCacheObj; +typedef struct SCacheObj SCacheObj; +typedef struct SCacheIter SCacheIter; +typedef struct STrashElem STrashElem; /** * initialize the cache object @@ -141,7 +95,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data); * @param data * @return */ -void *taosCacheTransfer(SCacheObj *pCacheObj, void **data); +void *taosCacheTransferData(SCacheObj *pCacheObj, void **data); /** * remove data in cache, the data will not be removed immediately. @@ -152,6 +106,13 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data); */ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove); +/** + * + * @param pCacheObj + * @return + */ +size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj); + /** * move all data node into trash, clear node in trash can if it is not referenced by any clients * @param handle @@ -184,6 +145,12 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1); */ void taosStopCacheRefreshWorker(); +SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj); +bool taosCacheIterNext(SCacheIter* pIter); +void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen); +void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen); +void taosCacheDestroyIter(SCacheIter* pIter); + #ifdef __cplusplus } #endif diff --git a/include/util/thash.h b/include/util/thash.h index 5c344f3f0f..57b20c65ee 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -28,12 +28,8 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *); -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) -#define HASH_DEFAULT_LOAD_FACTOR (0.75) - -#define HASH_INDEX(v, c) ((v) & ((c)-1)) - -#define HASH_NODE_EXIST(code) (code == -2) +#define HASH_KEY_ALREADY_EXISTS (-2) +#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS) /** * murmur hash algorithm @@ -54,49 +50,15 @@ uint32_t taosIntHash_32(const char *key, uint32_t len); uint32_t taosIntHash_64(const char *key, uint32_t len); _hash_fn_t taosGetDefaultHashFunction(int32_t type); - _equal_fn_t taosGetDefaultEqualFunction(int32_t type); -typedef struct SHashNode { - struct SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t dataLen; // length of data - uint32_t keyLen; // length of the key - uint16_t count; // reference count - int8_t removed; // flag to indicate removed - char data[]; -} SHashNode; - -#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen) -#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode)) -#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode))) - typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, } SHashLockTypeE; -typedef struct SHashEntry { - int32_t num; // number of elements in current entry - SRWLatch latch; // entry latch - SHashNode *next; -} SHashEntry; - -typedef struct SHashObj { - SHashEntry **hashList; - uint32_t capacity; // number of slots - uint32_t size; // number of elements in hash table - - _hash_fn_t hashFp; // hash function - _hash_free_fn_t freeFp; // hash node free callback function - _equal_fn_t equalFp; // equal function - _hash_before_fn_t callbackFp; // function invoked before return the value to caller - - SRWLatch lock; // read-write spin lock - SHashLockTypeE type; // lock type - bool enableUpdate; // enable update - SArray *pMemBlock; // memory block allocated for SHashEntry -} SHashObj; +typedef struct SHashNode SHashNode; +typedef struct SHashObj SHashObj; /** * init the hash table @@ -126,8 +88,6 @@ int32_t taosHashGetSize(const SHashObj *pHashObj); */ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size); -int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded); - /** * return the payload data with the specified key * @@ -146,17 +106,18 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); * @param destBuf * @return */ -void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf); +int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf); /** - * Clone the result to interval allocated buffer + * * @param pHashObj * @param key * @param keyLen * @param destBuf + * @param size * @return */ -void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, size_t *sz); +int32_t taosHashGetDup_m(SHashObj* pHashObj, const void* key, size_t keyLen, void** destBuf, int32_t* size); /** * remove item with the specified key @@ -207,37 +168,13 @@ void *taosHashIterate(SHashObj *pHashObj, void *p); */ void taosHashCancelIterate(SHashObj *pHashObj, void *p); -/** - * Get the corresponding key information for a given data in hash table - * @param data - * @return - */ -int32_t taosHashGetKey(void *data, void **key, size_t *keyLen); - -/** - * Get the corresponding key information for a given data in hash table, using memcpy - * @param data - * @param dst - * @return - */ -static FORCE_INLINE int32_t taosHashCopyKey(void *data, void *dst) { - if (NULL == data || NULL == dst) { - return -1; - } - - SHashNode *node = GET_HASH_PNODE(data); - void *key = GET_HASH_NODE_KEY(node); - memcpy(dst, key, node->keyLen); - - return 0; -} - -/** - * Get the corresponding data length for a given data in hash table - * @param data - * @return - */ -int32_t taosHashGetDataLen(void *data); + /** + * Get the corresponding key information for a given data in hash table + * @param data + * @param keyLen + * @return + */ +void *taosHashGetKey(void *data, size_t* keyLen); /** * return the payload data with the specified key(reference number added) @@ -258,8 +195,20 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); */ void taosHashRelease(SHashObj *pHashObj, void *p); +/** + * + * @param pHashObj + * @param fp + */ void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); +/** + * + * @param pHashObj + * @param fp + */ +void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp); + #ifdef __cplusplus } #endif diff --git a/include/util/tlog.h b/include/util/tlog.h index 6e6795e9a2..d3ab9b0bfb 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -22,9 +22,29 @@ extern "C" { #endif +typedef enum { + DEBUG_FATAL = 1, + DEBUG_ERROR = 1, + DEBUG_WARN = 2, + DEBUG_INFO = 2, + DEBUG_DEBUG = 4, + DEBUG_TRACE = 8, + DEBUG_DUMP = 16, + DEBUG_SCREEN = 64, + DEBUG_FILE = 128 +} ELogLevel; + +typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content); + +extern bool tsLogEmbedded; extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; +extern LogFp tsLogFp; +extern int64_t tsNumOfErrorLogs; +extern int64_t tsNumOfInfoLogs; +extern int64_t tsNumOfDebugLogs; +extern int64_t tsNumOfTraceLogs; extern int32_t dDebugFlag; extern int32_t vDebugFlag; extern int32_t mDebugFlag; @@ -40,45 +60,33 @@ extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; extern int32_t fsDebugFlag; -#define DEBUG_FATAL 1U -#define DEBUG_ERROR DEBUG_FATAL -#define DEBUG_WARN 2U -#define DEBUG_INFO DEBUG_WARN -#define DEBUG_DEBUG 4U -#define DEBUG_TRACE 8U -#define DEBUG_DUMP 16U -#define DEBUG_SCREEN 64U -#define DEBUG_FILE 128U - int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); void taosResetLog(); void taosSetAllDebugFlag(int32_t flag); void taosDumpData(uint8_t *msg, int32_t len); -void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) +void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) #ifdef __GNUC__ - __attribute__((format(printf, 3, 4))) + __attribute__((format(printf, 4, 5))) #endif ; -void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) +void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) #ifdef __GNUC__ - __attribute__((format(printf, 3, 4))) + __attribute__((format(printf, 4, 5))) #endif ; -extern int8_t tscEmbeddedInUtil; +#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", DEBUG_INFO, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }} +#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }} -#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} -#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} - -#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } -#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); } +#define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } +#define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } #ifdef __cplusplus } diff --git a/include/util/tpagedbuf.h b/include/util/tpagedbuf.h index ce9a57c2c3..acaff759b7 100644 --- a/include/util/tpagedbuf.h +++ b/include/util/tpagedbuf.h @@ -53,7 +53,7 @@ typedef struct SDiskbasedBufStatis { * @param handle * @return */ -int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir); /** * diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c93ea1aabe..0e1b564b03 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,12 +20,12 @@ extern "C" { #endif -#include "tcommon.h" #include "parser.h" #include "query.h" #include "taos.h" +#include "tcommon.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "thash.h" #include "tlist.h" #include "tmsg.h" @@ -47,12 +47,12 @@ extern "C" { typedef struct SAppInstInfo SAppInstInfo; -typedef struct SHbConnInfo { +typedef struct { void* param; SClientHbReq* req; } SHbConnInfo; -typedef struct SAppHbMgr { +typedef struct { char* key; // statistics int32_t reportCnt; @@ -68,11 +68,11 @@ typedef struct SAppHbMgr { SHashObj* connInfo; // hash } SAppHbMgr; -typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); +typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); -typedef struct SClientHbMgr { +typedef struct { int8_t inited; // ctl int8_t threadStop; @@ -108,13 +108,13 @@ typedef struct SHeartBeatInfo { } SHeartBeatInfo; struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - SInstanceSummary summary; - SList* pConnList; // STscObj linked list - int64_t clusterId; - void* pTransporter; - struct SAppHbMgr* pAppHbMgr; + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceSummary summary; + SList* pConnList; // STscObj linked list + int64_t clusterId; + void* pTransporter; + SAppHbMgr* pAppHbMgr; }; typedef struct SAppInfo { @@ -141,10 +141,6 @@ typedef struct STscObj { SAppInstInfo* pAppInfo; } STscObj; -typedef struct SMqConsumer { - STscObj* pTscObj; -} SMqConsumer; - typedef struct SReqResultInfo { const char* pRspMsg; const char* pData; @@ -172,7 +168,7 @@ typedef struct SRequestSendRecvBody { SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; int64_t queryJob; // query job, created according to sql query DAG. - struct SQueryDag* pDag; // the query dag, generated according to the sql statement. + struct SQueryDag* pDag; // the query dag, generated according to the sql statement. SReqResultInfo resInfo; } SRequestSendRecvBody; diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index bfa2c0319b..d47edcd795 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -22,13 +22,13 @@ extern "C" { #include "tlog.h" -#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index e9e64a78b8..b1e9de8000 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -23,7 +23,7 @@ static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); -static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } +static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -104,7 +104,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo return TSDB_CODE_SUCCESS; } -static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { +static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); if (NULL == info) { tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType); @@ -163,7 +163,7 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRs return TSDB_CODE_SUCCESS; } -static int32_t hbMqAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { +static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { static int32_t emptyRspNum = 0; if (code != 0) { tfree(param); @@ -226,7 +226,11 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl db->vgVersion = htonl(db->vgVersion); } - SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = sizeof(SDbVgVersion) * dbNum, .value = dbs}; + SKv kv = { + .key = HEARTBEAT_KEY_DBINFO, + .valueLen = sizeof(SDbVgVersion) * dbNum, + .value = dbs, + }; tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen); @@ -256,7 +260,11 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC stb->tversion = htons(stb->tversion); } - SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs}; + SKv kv = { + .key = HEARTBEAT_KEY_STBINFO, + .valueLen = sizeof(SSTableMetaVersion) * stbNum, + .value = stbs, + }; tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen); @@ -288,7 +296,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req return TSDB_CODE_SUCCESS; } -int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) {} +int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; } void hbMgrInitMqHbHandle() { clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle; @@ -396,7 +404,7 @@ static void *hbThreadFunc(void *param) { free(buf); break; } - pInfo->fp = hbMqAsyncCallBack; + pInfo->fp = hbAsyncCallBack; pInfo->msgInfo.pData = buf; pInfo->msgInfo.len = tlen; pInfo->msgType = TDMT_MND_HEARTBEAT; @@ -448,7 +456,6 @@ static void hbStopThread() { } SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { - /*return NULL;*/ hbMgrInit(); SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { @@ -473,7 +480,8 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { free(pAppHbMgr); return NULL; } - pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq; + + taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq); // init getInfoFunc pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); @@ -506,7 +514,6 @@ void appHbMgrCleanup(void) { } int hbMgrInit() { - /*return 0;*/ // init once int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); if (old == 1) return 0; @@ -524,7 +531,7 @@ int hbMgrInit() { } void hbMgrCleanUp() { - return; +#if 0 hbStopThread(); // destroy all appHbMgr @@ -537,6 +544,7 @@ void hbMgrCleanUp() { pthread_mutex_unlock(&clientHbMgr.lock); clientHbMgr.appHbMgrs = NULL; +#endif } int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { @@ -563,9 +571,11 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { - /*return 0;*/ - SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; - SHbConnInfo info = {0}; + SClientHbKey connKey = { + .connId = connId, + .hbType = HEARTBEAT_TYPE_QUERY, + }; + SHbConnInfo info = {0}; switch (hbType) { case HEARTBEAT_TYPE_QUERY: { @@ -586,7 +596,6 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3 } void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { - /*return;*/ int32_t code = 0; code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); @@ -598,7 +607,6 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen, int32_t valueLen) { - return 0; // find req by connection id SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); ASSERT(pReq != NULL); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a0ba668f8a..259d0e5799 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -4,8 +4,8 @@ #include "parser.h" #include "planner.h" #include "scheduler.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "tglobal.h" #include "tmsgtype.h" #include "tpagedbuf.h" diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index d9ab23b9fa..35fc557eea 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -20,8 +20,8 @@ #include "parser.h" #include "planner.h" #include "scheduler.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "tglobal.h" #include "tmsgtype.h" #include "tpagedbuf.h" @@ -59,6 +59,7 @@ struct tmq_t { char groupId[256]; char clientId[256]; int8_t autoCommit; + int8_t inWaiting; int64_t consumerId; int32_t epoch; int32_t resetOffsetCfg; @@ -66,9 +67,12 @@ struct tmq_t { STscObj* pTscObj; tmq_commit_cb* commit_cb; int32_t nextTopicIdx; + int32_t waitingRequest; + int32_t readyRequest; SArray* clientTopics; // SArray STaosQueue* mqueue; // queue of tmq_message_t STaosQall* qall; + tsem_t rspSem; // stat int64_t pollCnt; }; @@ -117,10 +121,12 @@ typedef struct { } SMqAskEpCbParam; typedef struct { - tmq_t* tmq; - SMqClientVg* pVg; - int32_t epoch; - tsem_t rspSem; + tmq_t* tmq; + SMqClientVg* pVg; + int32_t epoch; + tsem_t rspSem; + tmq_message_t** msg; + int32_t sync; } SMqPollCbParam; typedef struct { @@ -196,6 +202,26 @@ int32_t tmq_list_append(tmq_list_t* ptr, const char* src) { return 0; } +void tmqClearUnhandleMsg(tmq_t* tmq) { + tmq_message_t* msg; + while (1) { + taosGetQitem(tmq->qall, (void**)&msg); + if (msg) + taosFreeQitem(msg); + else + break; + } + + taosReadAllQitems(tmq->mqueue, tmq->qall); + while (1) { + taosGetQitem(tmq->qall, (void**)&msg); + if (msg) + taosFreeQitem(msg); + else + break; + } +} + int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; pParam->rspErr = code; @@ -205,16 +231,11 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; - tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; + pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; if (pParam->tmq->commit_cb) { - pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); - } - if (!pParam->async) - tsem_post(&pParam->rspSem); - else { - tsem_destroy(&pParam->rspSem); - free(param); + pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL, NULL); } + if (!pParam->async) tsem_post(&pParam->rspSem); return 0; } @@ -240,9 +261,12 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return NULL; } pTmq->pTscObj = (STscObj*)conn; + pTmq->inWaiting = 0; pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; + pTmq->waitingRequest = 0; + pTmq->readyRequest = 0; // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -250,6 +274,8 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->commit_cb = conf->commit_cb; pTmq->resetOffsetCfg = conf->resetOffset; + tsem_init(&pTmq->rspSem, 0, 0); + pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); @@ -315,6 +341,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in } pParam->tmq = tmq; tsem_init(&pParam->rspSem, 0, 0); + pParam->async = async; pRequest->body.requestMsg = (SDataBuf){ .pData = buf, @@ -335,6 +362,9 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in resp = pParam->rspErr; } + tsem_destroy(&pParam->rspSem); + free(pParam); + if (pArray) { taosArrayDestroy(pArray); } @@ -576,7 +606,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; - SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; return pRsp->skipLogNum; } @@ -625,56 +655,74 @@ void tmqShowMsg(tmq_message_t* tmq_message) { } int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { - printf("recv poll\n"); + /*printf("recv poll\n");*/ SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqClientVg* pVg = pParam->pVg; tmq_t* tmq = pParam->tmq; if (code != 0) { printf("msg discard\n"); - if (pParam->epoch == tmq->epoch) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - } - return 0; + goto WRITE_QUEUE_FAIL; } int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { + tsem_post(&tmq->rspSem); printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch); return 0; } if (msgEpoch != tmqEpoch) { printf("mismatch rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch); + } else { + atomic_sub_fetch_32(&tmq->waitingRequest, 1); } +#if 0 + if (pParam->sync == 1) { + /**pParam->msg = malloc(sizeof(tmq_message_t));*/ + *pParam->msg = taosAllocateQitem(sizeof(tmq_message_t)); + if (*pParam->msg) { + memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead)); + tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &((*pParam->msg)->consumeRsp)); + if ((*pParam->msg)->consumeRsp.numOfTopics != 0) { + pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset; + } + taosWriteQitem(tmq->mqueue, *pParam->msg); + tsem_post(&pParam->rspSem); + return 0; + } + tsem_post(&pParam->rspSem); + return -1; + } +#endif + /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/ tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); if (pRsp == NULL) { - printf("fail\n"); - return -1; + goto WRITE_QUEUE_FAIL; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->consumeRsp.numOfTopics == 0) { - printf("no data\n"); - if (pParam->epoch == tmq->epoch) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - } + /*printf("no data\n");*/ taosFreeQitem(pRsp); - return 0; + goto WRITE_QUEUE_FAIL; } + pRsp->extra = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); - printf("poll in queue\n"); - /*pParam->rspMsg = (tmq_message_t*)pRsp;*/ - /*pVg->currentOffset = pRsp->consumeRsp.rspOffset;*/ - - /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ - /*printf("-----msg begin----\n");*/ - /*printf("\n-----msg end------\n");*/ + atomic_add_fetch_32(&tmq->readyRequest, 1); + tsem_post(&tmq->rspSem); return 0; + +WRITE_QUEUE_FAIL: + if (pParam->epoch == tmq->epoch) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + tsem_post(&tmq->rspSem); + return code; } bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { @@ -711,81 +759,94 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tmq_t* tmq = pParam->tmq; if (code != 0) { printf("get topic endpoint error, not ready, wait:%d\n", pParam->sync); - if (pParam->sync) { - tsem_post(&pParam->rspSem); - } - return 0; + goto END; } - tscDebug("tmq ask ep cb called"); + + // tmq's epoch is monotomically increase, + // so it's safe to discard any old epoch msg. + // epoch will only increase when received newer epoch ep msg + SMqRspHead* head = pMsg->pData; + int32_t epoch = atomic_load_32(&tmq->epoch); + if (head->epoch <= epoch) { + goto END; + } + if (pParam->sync) { - SMqRspHead* head = pMsg->pData; SMqCMGetSubEpRsp rsp; tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ - int32_t epoch = atomic_load_32(&tmq->epoch); - if (head->epoch > epoch && tmqUpdateEp(tmq, head->epoch, &rsp)) { + if (tmqUpdateEp(tmq, head->epoch, &rsp)) { atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY); } - tsem_post(&pParam->rspSem); tDeleteSMqCMGetSubEpRsp(&rsp); } else { tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = -1; + goto END; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp); + taosWriteQitem(tmq->mqueue, pRsp); + tsem_post(&tmq->rspSem); } - return 0; + +END: + if (pParam->sync) { + tsem_post(&pParam->rspSem); + } + return code; } int32_t tmqAskEp(tmq_t* tmq, bool sync) { - printf("ask ep sync %d\n", sync); int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* buf = malloc(tlen); - if (buf == NULL) { + SMqCMGetSubEpReq* req = malloc(tlen); + if (req == NULL) { tscError("failed to malloc get subscribe ep buf"); - goto END; + return -1; } - buf->consumerId = htobe64(tmq->consumerId); - buf->epoch = htonl(tmq->epoch); - strcpy(buf->cgroup, tmq->groupId); - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); - if (pRequest == NULL) { - tscError("failed to malloc subscribe ep request"); - goto END; - } - - pRequest->body.requestMsg = (SDataBuf){ - .pData = buf, - .len = tlen, - .handle = NULL, - }; + req->consumerId = htobe64(tmq->consumerId); + req->epoch = htonl(tmq->epoch); + strcpy(req->cgroup, tmq->groupId); SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam)); if (pParam == NULL) { tscError("failed to malloc subscribe param"); - goto END; + free(req); + return -1; } pParam->tmq = tmq; pParam->sync = sync; tsem_init(&pParam->rspSem, 0, 0); - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + tsem_destroy(&pParam->rspSem); + free(pParam); + free(req); + return -1; + } + + sendInfo->msgInfo = (SDataBuf){ + .pData = req, + .len = tlen, + .handle = NULL, + }; + + sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqAskEpCb; + sendInfo->msgType = TDMT_MND_GET_SUB_EP; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); -END: if (sync) tsem_wait(&pParam->rspSem); return 0; } @@ -804,6 +865,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j); if (pVg->vgId == pOffset->vgId) { pVg->currentOffset = pOffset->offset; + tmqClearUnhandleMsg(tmq); return TMQ_RESP_ERR__SUCCESS; } } @@ -812,7 +874,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { return TMQ_RESP_ERR__FAIL; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) { +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { int64_t reqOffset; if (pVg->currentOffset >= 0) { reqOffset = pVg->currentOffset; @@ -832,7 +894,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie strcpy(pReq->topic, pTopic->topicName); strcpy(pReq->cgroup, tmq->groupId); - pReq->blockingTime = blocking_time; + pReq->blockingTime = blockingTime; pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; pReq->currentOffset = reqOffset; @@ -842,28 +904,76 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie return pReq; } -void tmqClearUnhandleMsg(tmq_t* tmq) { - tmq_message_t* msg; - while (1) { - taosGetQitem(tmq->qall, (void**)&msg); - if (msg) - taosFreeQitem(msg); - else - break; - } +tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { + tmq_message_t* msg = NULL; + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); + /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/ + /*continue;*/ + /*}*/ + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); + if (pReq == NULL) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + // TODO: out of mem + return NULL; + } - taosReadAllQitems(tmq->mqueue, tmq->qall); - while (1) { - taosGetQitem(tmq->qall, (void**)&msg); - if (msg) - taosFreeQitem(msg); - else - break; + SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + if (pParam == NULL) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + // TODO: out of mem + return NULL; + } + pParam->tmq = tmq; + pParam->pVg = pVg; + pParam->epoch = tmq->epoch; + pParam->sync = 1; + pParam->msg = &msg; + tsem_init(&pParam->rspSem, 0, 0); + + SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + return NULL; + } + + sendInfo->msgInfo = (SDataBuf){ + .pData = pReq, + .len = sizeof(SMqConsumeReq), + .handle = NULL, + }; + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqPollCb; + sendInfo->msgType = TDMT_VND_CONSUME; + + int64_t transporterId = 0; + /*printf("send poll\n");*/ + atomic_add_fetch_32(&tmq->waitingRequest, 1); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + pVg->pollCnt++; + tmq->pollCnt++; + + tsem_wait(&pParam->rspSem); + tmq_message_t* nmsg = NULL; + while (1) { + taosReadQitem(tmq->mqueue, (void**)&nmsg); + if (nmsg == NULL) continue; + while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) { + taosReadQitem(tmq->mqueue, (void**)&nmsg); + } + return nmsg; + } + } } + return NULL; } int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { - printf("call poll\n"); + /*printf("call poll\n");*/ for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { @@ -875,32 +985,44 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - // TODO: out of mem + tsem_post(&tmq->rspSem); return -1; } - SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); - if (param == NULL) { + SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam)); + if (pParam == NULL) { + free(pReq); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - // TODO: out of mem + tsem_post(&tmq->rspSem); return -1; } - param->tmq = tmq; - param->pVg = pVg; - param->epoch = tmq->epoch; - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){ + pParam->tmq = tmq; + pParam->pVg = pVg; + pParam->epoch = tmq->epoch; + pParam->sync = 0; + + SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + free(pReq); + free(pParam); + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + tsem_post(&tmq->rspSem); + return -1; + } + + sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq), .handle = NULL, }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; - sendInfo->param = param; + sendInfo->param = pParam; sendInfo->fp = tmqPollCb; + sendInfo->msgType = TDMT_VND_CONSUME; int64_t transporterId = 0; - printf("send poll\n"); + /*printf("send poll\n");*/ + atomic_add_fetch_32(&tmq->waitingRequest, 1); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; tmq->pollCnt++; @@ -912,7 +1034,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { // return int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) { if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) { - printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch); + /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/ if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) { tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp); tmqClearUnhandleMsg(tmq); @@ -931,23 +1053,26 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese tmq_message_t* rspMsg = NULL; taosGetQitem(tmq->qall, (void**)&rspMsg); if (rspMsg == NULL) { - break; + taosReadAllQitems(tmq->mqueue, tmq->qall); + taosGetQitem(tmq->qall, (void**)&rspMsg); + if (rspMsg == NULL) return NULL; } if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { - printf("handle poll rsp %d\n", rspMsg->head.mqMsgType); + atomic_sub_fetch_32(&tmq->readyRequest, 1); + /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) { - printf("epoch match\n"); + /*printf("epoch match\n");*/ SMqClientVg* pVg = rspMsg->extra; pVg->currentOffset = rspMsg->consumeRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); return rspMsg; } else { - printf("epoch mismatch\n"); + /*printf("epoch mismatch\n");*/ taosFreeQitem(rspMsg); } } else { - printf("handle ep rsp %d\n", rspMsg->head.mqMsgType); + /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/ bool reset = false; tmqHandleRes(tmq, rspMsg, &reset); taosFreeQitem(rspMsg); @@ -957,36 +1082,57 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese } } } - return NULL; +} + +tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { + tmq_message_t* rspMsg = NULL; + int64_t startTime = taosGetTimestampMs(); + + int64_t status = atomic_load_64(&tmq->status); + tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); + + while (1) { + rspMsg = tmqSyncPollImpl(tmq, blocking_time); + if (rspMsg && rspMsg->consumeRsp.numOfTopics) { + return rspMsg; + } + + if (blocking_time != 0) { + int64_t endTime = taosGetTimestampMs(); + if (endTime - startTime > blocking_time) { + return NULL; + } + } else + return NULL; + } } tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { - tmq_message_t* rspMsg = NULL; + tmq_message_t* rspMsg; int64_t startTime = taosGetTimestampMs(); // TODO: put into another thread or delayed queue int64_t status = atomic_load_64(&tmq->status); tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); - taosGetQitem(tmq->qall, (void**)&rspMsg); - if (rspMsg == NULL) { - taosReadAllQitems(tmq->mqueue, tmq->qall); + rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); + if (rspMsg) { + return rspMsg; } - tmqHandleAllRsp(tmq, blocking_time, false); - - tmqPollImpl(tmq, blocking_time); while (1) { /*printf("cycle\n");*/ - taosReadAllQitems(tmq->mqueue, tmq->qall); - rspMsg = tmqHandleAllRsp(tmq, blocking_time, true); + tmqPollImpl(tmq, blocking_time); + + tsem_wait(&tmq->rspSem); + + rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); if (rspMsg) { return rspMsg; } if (blocking_time != 0) { int64_t endTime = taosGetTimestampMs(); if (endTime - startTime > blocking_time) { - printf("normal exit\n"); return NULL; } } @@ -1127,6 +1273,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; tDeleteSMqConsumeRsp(pRsp); + /*free(tmq_message);*/ taosFreeQitem(tmq_message); } @@ -1138,6 +1285,7 @@ const char* tmq_err2str(tmq_resp_err_t err) { } return "fail"; } + #if 0 tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { tmq_t* pTmq = malloc(sizeof(tmq_t)); diff --git a/source/common/src/tep.c b/source/common/src/tdatablock.c similarity index 99% rename from source/common/src/tep.c rename to source/common/src/tdatablock.c index e2880441be..5b7557b749 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tdatablock.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "tep.h" +#include "tdatablock.h" #include "tcompare.h" #include "tglobal.h" diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5998b7c9ce..a521e10cc7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -17,7 +17,7 @@ #include "tglobal.h" #include "tcompare.h" #include "tconfig.h" -#include "tep.h" +#include "tdatablock.h" #include "tlog.h" SConfig *tsCfg = NULL; @@ -155,21 +155,40 @@ static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t prima uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); } -static void taosSetTfsCfg(SConfig *pCfg) { +static int32_t taosSetTfsCfg(SConfig *pCfg) { SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); - if (pItem == NULL) return; + memset(tsDataDir, 0, PATH_MAX); int32_t size = taosArrayGetSize(pItem->array); if (size <= 0) { tsDiskCfgNum = 1; taosAddDataDir(0, pItem->str, 0, 1); + tstrncpy(tsDataDir, pItem->str, PATH_MAX); + if (taosMkDir(tsDataDir) != 0) { + uError("failed to create dataDir:%s since %s", tsDataDir, terrstr()); + return -1; + } } else { tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS; for (int32_t index = 0; index < tsDiskCfgNum; ++index) { SDiskCfg *pCfg = taosArrayGet(pItem->array, index); memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); + if (pCfg->level == 0 && pCfg->primary == 1) { + tstrncpy(tsDataDir, pCfg->dir, PATH_MAX); + } + if (taosMkDir(pCfg->dir) != 0) { + uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr()); + return -1; + } } } + + if (tsDataDir[0] == 0) { + uError("datadir not set"); + return -1; + } + + return 0; } struct SConfig *taosGetCfg() { @@ -279,11 +298,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "totalMemory(MB)", tsTotalMemoryMB, 0, INT32_MAX, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "totalMemory(KB)", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1; if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; @@ -358,7 +377,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; } -static void taosSetClientCfg(SConfig *pCfg) { +static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); @@ -375,9 +394,13 @@ static void taosSetClientCfg(SConfig *pCfg) { snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); - tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); - taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); + tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + taosExpandDir(tsTempDir, tsTempDir, PATH_MAX); tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; + if (taosMkDir(tsTempDir) != 0) { + uError("failed to create tempDir:%s since %s", tsTempDir, terrstr()); + return -1; + } tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; @@ -392,6 +415,8 @@ static void taosSetClientCfg(SConfig *pCfg) { tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; + + return 0; } static void taosSetSystemCfg(SConfig *pCfg) { @@ -404,10 +429,6 @@ static void taosSetSystemCfg(SConfig *pCfg) { const char *charset = cfgGetItem(pCfg, "charset")->str; taosSetSystemLocale(locale, charset); - if (tsNumOfCores <= 1) { - tsNumOfCores = 2; - } - bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; taosSetConsoleEcho(enableCore); @@ -415,11 +436,8 @@ static void taosSetSystemCfg(SConfig *pCfg) { tsVersion = 30000000; } -static void taosSetServerCfg(SConfig *pCfg) { - tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); - taosExpandDir(tsDataDir, tsDataDir, PATH_MAX); - - tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; +static int32_t taosSetServerCfg(SConfig *pCfg) { + tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; @@ -448,6 +466,8 @@ static void taosSetServerCfg(SConfig *pCfg) { if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } + + return 0; } int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, @@ -458,10 +478,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi if (pCfg == NULL) return -1; if (tsc) { - tscEmbeddedInUtil = 0; + tsLogEmbedded = 0; if (taosAddClientLogCfg(pCfg) != 0) return -1; } else { - tscEmbeddedInUtil = 1; + tsLogEmbedded = 1; if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddServerLogCfg(pCfg) != 0) return -1; } @@ -508,8 +528,8 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU tsCfg = cfgInit(); if (tsc) { - if (taosAddClientLogCfg(tsCfg) != 0) return -1; if (taosAddClientCfg(tsCfg) != 0) return -1; + if (taosAddClientLogCfg(tsCfg) != 0) return -1; } else { if (taosAddClientCfg(tsCfg) != 0) return -1; if (taosAddServerCfg(tsCfg) != 0) return -1; @@ -532,24 +552,14 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU } if (tsc) { - taosSetClientCfg(tsCfg); + if (taosSetClientCfg(tsCfg)) return -1; } else { - taosSetClientCfg(tsCfg); - taosSetServerCfg(tsCfg); - taosSetTfsCfg(tsCfg); + if (taosSetClientCfg(tsCfg)) return -1; + if (taosSetServerCfg(tsCfg)) return -1; + if (taosSetTfsCfg(tsCfg) != 0) return -1; } taosSetSystemCfg(tsCfg); - if (taosMkDir(tsTempDir) != 0) { - uError("failed to create dir:%s since %s", tsTempDir, terrstr()); - return -1; - } - - if (!tsc && taosMkDir(tsDataDir) != 0) { - uError("failed to create dir:%s since %s", tsDataDir, terrstr()); - return -1; - } - cfgDumpCfg(tsCfg, tsc, false); return 0; } diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index f20e911681..464c29d287 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -46,7 +46,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } if (!autoDelete) { - remove(pTSBuf->path); + taosRemoveFile(pTSBuf->path); } if (NULL == allocResForTSBuf(pTSBuf)) { @@ -178,7 +178,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { if (pTSBuf->autoDelete) { // ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); - remove(pTSBuf->path); + taosRemoveFile(pTSBuf->path); } else { // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); } diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 4821d60875..7d54e5150c 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,6 +1,4 @@ -#include "tcommon.h" #include -#include #include #pragma GCC diagnostic push @@ -10,6 +8,9 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "tcommon.h" +#include "tdatablock.h" +#include "tcommon.h" #include "taos.h" #include "tvariant.h" #include "tdef.h" diff --git a/source/common/test/tmsgTest.cpp b/source/common/test/tmsgTest.cpp index ca33d24a8c..e8c3284ea5 100644 --- a/source/common/test/tmsgTest.cpp +++ b/source/common/test/tmsgTest.cpp @@ -1,6 +1,6 @@ #include -#include "gtest/gtest.h" +#include #include "tmsg.h" diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 724ba30155..b9acbea02f 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -31,7 +31,7 @@ typedef struct { SDnode *pDnode; STaosQueue *queue; union { - SQWorkerPool pool; + SQWorkerPool pool; SWWorkerPool mpool; }; } SDnodeWorker; @@ -108,6 +108,7 @@ typedef struct { SHashObj *hash; int32_t openVnodes; int32_t totalVnodes; + int32_t masterNum; SRWLatch latch; SQWorkerPool queryPool; SFWorkerPool fetchPool; @@ -137,8 +138,7 @@ typedef struct SDnode { SStartupReq startup; } SDnode; - -int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo); +int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 9fabe40186..4ca6b97ad4 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -23,9 +23,11 @@ extern "C" { #include "os.h" #include "cJSON.h" +#include "monitor.h" #include "tcache.h" #include "tcrc32c.h" -#include "tep.h" +#include "tdatablock.h" +#include "tglobal.h" #include "thash.h" #include "tlockfree.h" #include "tlog.h" @@ -35,8 +37,6 @@ extern "C" { #include "tthread.h" #include "ttime.h" #include "tworker.h" -#include "tglobal.h" -#include "monitor.h" #include "dnode.h" @@ -47,14 +47,12 @@ extern "C" { #include "vnode.h" #include "tfs.h" -extern int32_t dDebugFlag; - -#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} -#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} -#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }} -#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", 255, __VA_ARGS__); }} -#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} -#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} +#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; diff --git a/source/dnode/mgmt/impl/inc/dndMnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h index 0f03bb3832..0aee3b4b43 100644 --- a/source/dnode/mgmt/impl/inc/dndMnode.h +++ b/source/dnode/mgmt/impl/inc/dndMnode.h @@ -34,6 +34,7 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg); int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonGrantInfo *pGrantInfo); +int8_t dndIsMnode(SDnode *pDnode); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index ae08d6387c..1cbdf0c1dc 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -324,4 +324,11 @@ void dndCleanup() { dInfo("dnode env is cleaned up"); } -int32_t dndGetDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { return 0; } \ No newline at end of file +int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { + tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); + pInfo->logdir.size = tsLogSpace.size; + tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); + pInfo->tempdir.size = tsTempSpace.size; + + return tfsGetMonitorInfo(pDnode->pTfs, pInfo); +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 268ed7e362..ff9eb5d884 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -474,13 +474,36 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { rpcSendResponse(&rpcRsp); } -static int32_t dndGetBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { +static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { pInfo->dnode_id = dndGetDnodeId(pDnode); tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); - return 0; } -static int32_t dndGetDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { return 0; } +static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { + pInfo->uptime = (taosGetTimestampMs() - pDnode->dmgmt.rebootTime) / (86400000.0f); + taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); + pInfo->cpu_cores = tsNumOfCores; + taosGetProcMemory(&pInfo->mem_engine); + taosGetSysMemory(&pInfo->mem_system); + pInfo->mem_total = tsTotalMemoryKB; + pInfo->disk_engine = 0; + pInfo->disk_used = tsDataSpace.size.used; + pInfo->disk_total = tsDataSpace.size.total; + taosGetBandSpeed(&pInfo->net_in, &pInfo->net_out); + taosGetIOSpeed(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); + pInfo->req_select = 0; + pInfo->req_select_rate = 0; + pInfo->req_insert = 0; + pInfo->req_insert_success = 0; + pInfo->req_insert_rate = 0; + pInfo->req_insert_batch = 0; + pInfo->req_insert_batch_success = 0; + pInfo->req_insert_batch_rate = 0; + pInfo->errors = tsNumOfErrorLogs; + pInfo->vnodes_num = pDnode->vmgmt.totalVnodes; + pInfo->masters = pDnode->vmgmt.masterNum; + pInfo->has_mnode = dndIsMnode(pDnode); +} static void dndSendMonitorReport(SDnode *pDnode) { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; @@ -490,9 +513,8 @@ static void dndSendMonitorReport(SDnode *pDnode) { if (pMonitor == NULL) return; SMonBasicInfo basicInfo = {0}; - if (dndGetBasicInfo(pDnode, &basicInfo) == 0) { - monSetBasicInfo(pMonitor, &basicInfo); - } + dndGetMonitorBasicInfo(pDnode, &basicInfo); + monSetBasicInfo(pMonitor, &basicInfo); SMonClusterInfo clusterInfo = {0}; SMonVgroupInfo vgroupInfo = {0}; @@ -504,15 +526,19 @@ static void dndSendMonitorReport(SDnode *pDnode) { } SMonDnodeInfo dnodeInfo = {0}; - if (dndGetDnodeInfo(pDnode, &dnodeInfo) == 0) { - monSetDnodeInfo(pMonitor, &dnodeInfo); - } + dndGetMonitorDnodeInfo(pDnode, &dnodeInfo); + monSetDnodeInfo(pMonitor, &dnodeInfo); SMonDiskInfo diskInfo = {0}; - if (dndGetDiskInfo(pDnode, &diskInfo) == 0) { + if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { monSetDiskInfo(pMonitor, &diskInfo); } + taosArrayDestroy(clusterInfo.dnodes); + taosArrayDestroy(clusterInfo.mnodes); + taosArrayDestroy(vgroupInfo.vgroups); + taosArrayDestroy(diskInfo.datadirs); + monSendReport(pMonitor); monCleanupMonitorInfo(pMonitor); } diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 6cb117867f..47e74b5c57 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -639,4 +639,11 @@ int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SM int32_t code = mndGetMonitorInfo(pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); dndReleaseMnode(pDnode, pMnode); return code; +} + +int8_t dndIsMnode(SDnode *pDnode) { + SMnode *pMnode = dndAcquireMnode(pDnode); + if (pMnode == NULL) return 0; + dndReleaseMnode(pDnode, pMnode); + return 1; } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 2767461f12..28bc615aba 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -17,6 +17,7 @@ #include "dndVnodes.h" #include "dndMgmt.h" #include "dndTransport.h" +#include "sync.h" typedef struct { int32_t vgId; @@ -85,7 +86,7 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { int32_t refCount = 0; taosRLockLatch(&pMgmt->latch); - taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); if (pVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; } else { @@ -979,6 +980,8 @@ void dndCleanupVnodes(SDnode *pDnode) { void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; + int32_t totalVnodes = 0; + int32_t masterNum = 0; taosRLockLatch(&pMgmt->latch); @@ -993,8 +996,12 @@ void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) { vnodeGetLoad(pVnode->pImpl, &vload); taosArrayPush(pLoads, &vload); + totalVnodes++; + if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++; pIter = taosHashIterate(pMgmt->hash, pIter); } taosRUnLockLatch(&pMgmt->latch); + pMgmt->totalVnodes = totalVnodes; + pMgmt->masterNum = masterNum; } diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index b89cb02834..f22bc9d276 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -14,7 +14,7 @@ */ #include "sut.h" -#include "tep.h" +#include "tdatablock.h" static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { TestClient* client = (TestClient*)parent; diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 00a79ebfd9..2e38ea7513 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -28,7 +28,7 @@ void Testbase::InitLog(const char* path) { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - tscEmbeddedInUtil = 1; + tsLogEmbedded = 1; tsAsyncLog = 0; taosRemoveDir(path); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index bb2367ef72..91d37c4eed 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -34,46 +34,6 @@ extern "C" { #endif -extern int32_t mDebugFlag; - -// mnode log function -#define mFatal(...) \ - { \ - if (mDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("MND FATAL ", 255, __VA_ARGS__); \ - } \ - } -#define mError(...) \ - { \ - if (mDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("MND ERROR ", 255, __VA_ARGS__); \ - } \ - } -#define mWarn(...) \ - { \ - if (mDebugFlag & DEBUG_WARN) { \ - taosPrintLog("MND WARN ", 255, __VA_ARGS__); \ - } \ - } -#define mInfo(...) \ - { \ - if (mDebugFlag & DEBUG_INFO) { \ - taosPrintLog("MND ", 255, __VA_ARGS__); \ - } \ - } -#define mDebug(...) \ - { \ - if (mDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \ - } \ - } -#define mTrace(...) \ - { \ - if (mDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \ - } \ - } - typedef enum { MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_USER, diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 929d47184b..0fe24cd757 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -20,7 +20,7 @@ #include "sdb.h" #include "tcache.h" -#include "tep.h" +#include "tdatablock.h" #include "tglobal.h" #include "tqueue.h" #include "ttime.h" @@ -31,6 +31,13 @@ extern "C" { #endif +#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} +#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} + typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); @@ -85,6 +92,11 @@ typedef struct { ESyncState state; } SSyncMgmt; +typedef struct { + int64_t expireTimeMS; + int64_t timeseriesAllowed; +} SGrantInfo; + typedef struct SMnode { int32_t dnodeId; int64_t clusterId; @@ -105,6 +117,7 @@ typedef struct SMnode { STelemMgmt telemMgmt; SSyncMgmt syncMgmt; SHashObj *infosMeta; + SGrantInfo grant; MndMsgFp msgFp[TDMT_MAX]; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; @@ -120,7 +133,7 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); uint64_t mndGenerateUid(char *name, int32_t len); -int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); +void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndProfile.h b/source/dnode/mnode/impl/inc/mndProfile.h index df857be73e..d5954e1ac6 100644 --- a/source/dnode/mnode/impl/inc/mndProfile.h +++ b/source/dnode/mnode/impl/inc/mndProfile.h @@ -24,6 +24,7 @@ extern "C" { int32_t mndInitProfile(SMnode *pMnode); void mndCleanupProfile(SMnode *pMnode); +int32_t mndGetNumOfConnections(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index cbb31bc504..9a6297a0f4 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -99,11 +99,15 @@ void mndUpdateMnodeRole(SMnode *pMnode) { pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); if (pIter == NULL) break; + ESyncState lastRole = pObj->role; if (pObj->id == 1) { pObj->role = TAOS_SYNC_STATE_LEADER; } else { pObj->role = TAOS_SYNC_STATE_CANDIDATE; } + if (pObj->role != lastRole) { + pObj->roleTime = taosGetTimestampMs(); + } sdbRelease(pSdb, pObj); } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e313c4d676..dbf299e8da 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -48,7 +48,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); -static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn); +static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq); static int32_t mndProcessConnectReq(SMnodeMsg *pReq); @@ -158,27 +158,23 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { taosCacheRelease(pMgmt->cache, (void **)&pConn, false); } -static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - - *pConn = NULL; - - pIter = taosHashIterate(pMgmt->cache->pHashTable, pIter); - if (pIter == NULL) return NULL; - - SCacheDataNode **pNode = pIter; - if (pNode == NULL || *pNode == NULL) { - taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); - return NULL; +void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) { + SConnObj* pConn = NULL; + bool hasNext = taosCacheIterNext(pIter); + if (hasNext) { + size_t dataLen = 0; + pConn = taosCacheIterGetData(pIter, &dataLen); + } else { + taosCacheDestroyIter(pIter); } - *pConn = (SConnObj *)((*pNode)->data); - return pIter; + return pConn; } static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); + if (pIter != NULL) { + taosCacheDestroyIter(pIter); + } } static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { @@ -376,8 +372,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { int32_t rspLen = 0; mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { - SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; - taosArrayPush(hbRsp.info, &kv); + SKv kv1 = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); } break; } @@ -386,8 +382,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { int32_t rspLen = 0; mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { - SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; - taosArrayPush(hbRsp.info, &kv); + SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); } break; } @@ -638,7 +634,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } - pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); + pShow->numOfRows = taosCacheGetNumOfObj(pMgmt->cache); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; strcpy(pMeta->tbName, mndShowStr(pShow->type)); @@ -653,8 +649,13 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in char *pWrite; char ipStr[TSDB_IPv4ADDR_LEN + 6]; + if (pShow->pIter == NULL) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + pShow->pIter = taosCacheCreateIter(pMgmt->cache); + } + while (numOfRows < rows) { - pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); + pConn = mndGetNextConn(pMnode, pShow->pIter); if (pConn == NULL) break; cols = 0; @@ -823,19 +824,24 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, void *pIter; char str[TSDB_IPv4ADDR_LEN + 6] = {0}; + if (pShow->pIter == NULL) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + pShow->pIter = taosCacheCreateIter(pMgmt->cache); + } + while (numOfRows < rows) { - pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); + pConn = mndGetNextConn(pMnode, pShow->pIter); if (pConn == NULL) { - pShow->pIter = pIter; + pShow->pIter = NULL; break; } if (numOfRows + pConn->numOfQueries >= rows) { - mndCancelGetNextConn(pMnode, pIter); + taosCacheDestroyIter(pShow->pIter); + pShow->pIter = NULL; break; } - pShow->pIter = pIter; for (int32_t i = 0; i < pConn->numOfQueries; ++i) { SQueryDesc *pDesc = pConn->pQueries + i; cols = 0; @@ -913,6 +919,12 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, } static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); + if (pIter != NULL) { + taosCacheDestroyIter(pIter); + } } + +int32_t mndGetNumOfConnections(SMnode *pMnode) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + return taosCacheGetNumOfObj(pMgmt->cache); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 1cd7593846..0e2141a4d7 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -18,15 +18,15 @@ #include "mndCluster.h" #include "mndSync.h" #include "tbuffer.h" -#include "tjson.h" #include "thttp.h" +#include "tjson.h" #define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_PORT 80 static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) { SMnodeLoad load = {0}; - if (mndGetLoad(pMnode, &load) != 0) return; + mndGetLoad(pMnode, &load); tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode); tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode); @@ -52,23 +52,20 @@ static char* mndBuildTelemetryReport(SMnode* pMnode) { tjsonAddStringToObject(pJson, "instanceId", clusterName); tjsonAddDoubleToObject(pJson, "reportVersion", 1); - if (taosGetOsReleaseName(tmp, sizeof(tmp))) { + if (taosGetOsReleaseName(tmp, sizeof(tmp)) == 0) { tjsonAddStringToObject(pJson, "os", tmp); } - int32_t numOfCores = 0; - if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores)) { + float numOfCores = 0; + if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { tjsonAddStringToObject(pJson, "cpuModel", tmp); tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); } else { - tjsonAddDoubleToObject(pJson, "numOfCpu", taosGetCpuCores()); + tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores); } - uint64_t memoryKB = 0; - if (taosGetTotalSysMemoryKB(&memoryKB)) { - snprintf(tmp, sizeof(tmp), "%" PRIu64 " kB", memoryKB); - tjsonAddStringToObject(pJson, "memory", tmp); - } + snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); + tjsonAddStringToObject(pJson, "memory", tmp); tjsonAddStringToObject(pJson, "version", version); tjsonAddStringToObject(pJson, "buildInfo", buildinfo); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index c95857a2d4..64c7a66bf9 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -359,6 +359,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } + mndUpdateMnodeRole(pMnode); mDebug("mnode open successfully "); return pMnode; } @@ -385,26 +386,6 @@ void mndDestroy(const char *path) { mDebug("mnode is destroyed"); } -int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { - pLoad->numOfDnode = 0; - pLoad->numOfMnode = 0; - pLoad->numOfVgroup = 0; - pLoad->numOfDatabase = 0; - pLoad->numOfSuperTable = 0; - pLoad->numOfChildTable = 0; - pLoad->numOfColumn = 0; - pLoad->totalPoints = 0; - pLoad->totalStorage = 0; - pLoad->compStorage = 0; - - return 0; -} - -int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo) { - return 0; - } - SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); if (pMsg == NULL) { @@ -523,3 +504,131 @@ uint64_t mndGenerateUid(char *name, int32_t len) { } } while (true); } + +void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { + memset(pLoad, 0, sizeof(SMnodeLoad)); + + SSdb *pSdb = pMnode->pSdb; + pLoad->numOfDnode = sdbGetSize(pSdb, SDB_DNODE); + pLoad->numOfMnode = sdbGetSize(pSdb, SDB_MNODE); + pLoad->numOfVgroup = sdbGetSize(pSdb, SDB_VGROUP); + pLoad->numOfDatabase = sdbGetSize(pSdb, SDB_DB); + pLoad->numOfSuperTable = sdbGetSize(pSdb, SDB_STB); + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + pLoad->numOfChildTable += pVgroup->numOfTables; + pLoad->numOfColumn += pVgroup->numOfTimeSeries; + pLoad->totalPoints += pVgroup->pointsWritten; + pLoad->totalStorage += pVgroup->totalStorage; + pLoad->compStorage += pVgroup->compStorage; + + sdbRelease(pSdb, pVgroup); + } +} + +int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo) { + if (!mndIsMaster(pMnode)) return -1; + + SSdb *pSdb = pMnode->pSdb; + int64_t ms = taosGetTimestampMs(); + + pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc)); + pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc)); + pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc)); + if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL) { + return -1; + } + + // cluster info + tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version)); + pClusterInfo->monitor_interval = tsMonitorInterval; + pClusterInfo->connections_total = mndGetNumOfConnections(pMnode); + + void *pIter = NULL; + while (1) { + SDnodeObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SMonDnodeDesc desc = {0}; + desc.dnode_id = pObj->id; + tstrncpy(desc.dnode_ep, pObj->ep, sizeof(desc.dnode_ep)); + if (mndIsDnodeOnline(pMnode, pObj, ms)) { + tstrncpy(desc.status, "ready", sizeof(desc.status)); + } else { + tstrncpy(desc.status, "offline", sizeof(desc.status)); + } + taosArrayPush(pClusterInfo->dnodes, &desc); + sdbRelease(pSdb, pObj); + } + + pIter = NULL; + while (1) { + SMnodeObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SMonMnodeDesc desc = {0}; + desc.mnode_id = pObj->id; + tstrncpy(desc.mnode_ep, pObj->pDnode->ep, sizeof(desc.mnode_ep)); + tstrncpy(desc.role, mndGetRoleStr(pObj->role), sizeof(desc.role)); + taosArrayPush(pClusterInfo->mnodes, &desc); + sdbRelease(pSdb, pObj); + + if (pObj->role == TAOS_SYNC_STATE_LEADER) { + pClusterInfo->first_ep_dnode_id = pObj->id; + tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); + pClusterInfo->master_uptime = (ms - pObj->roleTime) / (86400000.0f); + } + } + + // vgroup info + pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + pClusterInfo->vgroups_total++; + + SMonVgroupDesc desc = {0}; + desc.vgroup_id = pVgroup->vgId; + strncpy(desc.database_name, pVgroup->dbName, sizeof(desc.database_name)); + desc.tables_num = pVgroup->numOfTables; + pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries; + tstrncpy(desc.status, "unsynced", sizeof(desc.status)); + for (int32_t i = 0; i < pVgroup->replica; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SMonVnodeDesc *pVnDesc = &desc.vnodes[i]; + pVnDesc->dnode_id = pVgid->dnodeId; + tstrncpy(pVnDesc->vnode_role, mndGetRoleStr(pVgid->role), sizeof(pVnDesc->vnode_role)); + if (pVgid->role == TAOS_SYNC_STATE_LEADER) { + tstrncpy(desc.status, "ready", sizeof(desc.status)); + pClusterInfo->vgroups_alive++; + } + if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_CANDIDATE) { + pClusterInfo->vnodes_alive++; + } + pClusterInfo->vnodes_total++; + } + + taosArrayPush(pVgroupInfo->vgroups, &desc); + sdbRelease(pSdb, pVgroup); + } + + // grant info + pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f; + pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; + if (pMnode->grant.expireTimeMS == 0) { + pGrantInfo->expire_time = INT32_MAX; + pGrantInfo->timeseries_total = INT32_MAX; + } + + return 0; +} \ No newline at end of file diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index c99dff57e1..d268b93051 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -28,12 +28,12 @@ extern "C" { #endif -#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }} -#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }} -#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }} -#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", 255, __VA_ARGS__); }} -#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} -#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} +#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} +#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} #define SDB_MAX_SIZE (32 * 1024) diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index c98a97d0dc..fd079b8f32 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -24,8 +24,8 @@ extern "C" { #endif -#define META_SUPER_TABLE TD_SUPER_TABLE -#define META_CHILD_TABLE TD_CHILD_TABLE +#define META_SUPER_TABLE TD_SUPER_TABLE +#define META_CHILD_TABLE TD_CHILD_TABLE #define META_NORMAL_TABLE TD_NORMAL_TABLE // Types exported @@ -50,14 +50,14 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); // For Query -STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); -STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); +STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); +STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); -STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); +STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); -char * metaTbCursorNext(SMTbCursor *pTbCur); +char *metaTbCursorNext(SMTbCursor *pTbCur); SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid); void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7718e0886d..319bd9fde6 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -213,6 +213,10 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA //} static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { + if (pHandle->tbIdHash) { + taosHashClear(pHandle->tbIdHash); + } + pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -227,6 +231,23 @@ static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const S return 0; } +static FORCE_INLINE int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { + if (pHandle->tbIdHash == NULL) { + pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pHandle->tbIdHash == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { + int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i); + taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); + } + + return 0; +} + int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); diff --git a/source/dnode/vnode/src/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h index f4128f7170..71bfd91356 100644 --- a/source/dnode/vnode/src/inc/metaDef.h +++ b/source/dnode/vnode/src/inc/metaDef.h @@ -76,4 +76,4 @@ struct SMeta { } #endif -#endif /*_TD_META_DEF_H_*/ \ No newline at end of file +#endif /*_TD_META_DEF_H_*/ diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 30a83ca634..f416413859 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -25,43 +25,41 @@ extern "C" { #endif -extern int32_t tqDebugFlag; - -#define tqFatal(...) \ - { \ - if (tqDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); \ - } \ +#define tqFatal(...) \ + { \ + if (tqDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } -#define tqError(...) \ - { \ - if (tqDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); \ - } \ +#define tqError(...) \ + { \ + if (tqDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } -#define tqWarn(...) \ - { \ - if (tqDebugFlag & DEBUG_WARN) { \ - taosPrintLog("TQ WARN ", 255, __VA_ARGS__); \ - } \ +#define tqWarn(...) \ + { \ + if (tqDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } -#define tqInfo(...) \ - { \ - if (tqDebugFlag & DEBUG_INFO) { \ - taosPrintLog("TQ ", 255, __VA_ARGS__); \ - } \ +#define tqInfo(...) \ + { \ + if (tqDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } -#define tqDebug(...) \ - { \ - if (tqDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ - } \ +#define tqDebug(...) \ + { \ + if (tqDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \ + } \ } -#define tqTrace(...) \ - { \ - if (tqDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ - } \ +#define tqTrace(...) \ + { \ + if (tqDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); \ + } \ } #define TQ_BUFFER_SIZE 8 diff --git a/source/dnode/vnode/src/inc/tsdbLog.h b/source/dnode/vnode/src/inc/tsdbLog.h index bde9b338a2..6ab17ec587 100644 --- a/source/dnode/vnode/src/inc/tsdbLog.h +++ b/source/dnode/vnode/src/inc/tsdbLog.h @@ -20,11 +20,11 @@ extern int32_t tsdbDebugFlag; -#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) -#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) -#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) -#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) -#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) #endif /* _TD_TSDB_LOG_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 75088687a8..6f4f0049e3 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -88,44 +88,41 @@ int vnodeScheduleTask(SVnodeTask* task); int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); -// For Log -extern int32_t vDebugFlag; - -#define vFatal(...) \ - do { \ - if (vDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("VND FATAL ", 255, __VA_ARGS__); \ - } \ +#define vFatal(...) \ + do { \ + if (vDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("VND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } while (0) -#define vError(...) \ - do { \ - if (vDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("VND ERROR ", 255, __VA_ARGS__); \ - } \ +#define vError(...) \ + do { \ + if (vDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("VND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } while (0) -#define vWarn(...) \ - do { \ - if (vDebugFlag & DEBUG_WARN) { \ - taosPrintLog("VND WARN ", 255, __VA_ARGS__); \ - } \ +#define vWarn(...) \ + do { \ + if (vDebugFlag & DEBUG_WARN) { \ + taosPrintLog("VND WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } while (0) -#define vInfo(...) \ - do { \ - if (vDebugFlag & DEBUG_INFO) { \ - taosPrintLog("VND ", 255, __VA_ARGS__); \ - } \ +#define vInfo(...) \ + do { \ + if (vDebugFlag & DEBUG_INFO) { \ + taosPrintLog("VND ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } while (0) -#define vDebug(...) \ - do { \ - if (vDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \ - } \ +#define vDebug(...) \ + do { \ + if (vDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("VND ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define vTrace(...) \ - do { \ - if (vDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \ - } \ +#define vTrace(...) \ + do { \ + if (vDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("VND ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); \ + } \ } while (0) // vnodeCfg.h diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index 3c464391d2..f75d9d3db0 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -62,10 +62,10 @@ static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT * static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); -static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); +static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); -static void * metaDecodeSchema(void *buf, SSchemaWrapper *pSW); +static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW); static void metaDBWLock(SMetaDB *pDB); static void metaDBRLock(SMetaDB *pDB); static void metaDBULock(SMetaDB *pDB); @@ -142,7 +142,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { tb_uid_t uid; char buf[512]; char buf1[512]; - void * pBuf; + void *pBuf; DBT key1, value1; DBT key2, value2; SSchema *pSchema = NULL; @@ -394,7 +394,7 @@ static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); - DBT * pDbt; + DBT *pDbt; if (pTbCfg->type == META_CHILD_TABLE) { // pDbt = calloc(2, sizeof(DBT)); @@ -479,7 +479,7 @@ static void metaClearTbCfg(STbCfg *pTbCfg) { /* ------------------------ FOR QUERY ------------------------ */ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { - STbCfg * pTbCfg = NULL; + STbCfg *pTbCfg = NULL; SMetaDB *pDB = pMeta->pDB; DBT key = {0}; DBT value = {0}; @@ -509,7 +509,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { } STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { - STbCfg * pTbCfg = NULL; + STbCfg *pTbCfg = NULL; SMetaDB *pDB = pMeta->pDB; DBT key = {0}; DBT pkey = {0}; @@ -543,10 +543,10 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { uint32_t nCols; SSchemaWrapper *pSW = NULL; - SMetaDB * pDB = pMeta->pDB; + SMetaDB *pDB = pMeta->pDB; int ret; - void * pBuf; - SSchema * pSchema; + void *pBuf; + SSchema *pSchema; SSchemaKey schemaKey = {uid, sver, 0}; DBT key = {0}; DBT value = {0}; @@ -578,7 +578,7 @@ struct SMTbCursor { SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { SMTbCursor *pTbCur = NULL; - SMetaDB * pDB = pMeta->pDB; + SMetaDB *pDB = pMeta->pDB; pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur)); if (pTbCur == NULL) { @@ -609,7 +609,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { DBT key = {0}; DBT value = {0}; STbCfg tbCfg; - void * pBuf; + void *pBuf; for (;;) { if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) { @@ -631,10 +631,10 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { STSchemaBuilder sb; - STSchema * pTSchema = NULL; - SSchema * pSchema; + STSchema *pTSchema = NULL; + SSchema *pSchema; SSchemaWrapper *pSW; - STbCfg * pTbCfg; + STbCfg *pTbCfg; tb_uid_t quid; pTbCfg = metaGetTbInfoByUid(pMeta, uid); @@ -662,13 +662,13 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { } struct SMCtbCursor { - DBC * pCur; + DBC *pCur; tb_uid_t suid; }; SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMCtbCursor *pCtbCur = NULL; - SMetaDB * pDB = pMeta->pDB; + SMetaDB *pDB = pMeta->pDB; int ret; pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur)); @@ -700,7 +700,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { DBT skey = {0}; DBT pkey = {0}; DBT pval = {0}; - void * pBuf; + void *pBuf; STbCfg tbCfg; // Set key diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index aa198d0806..ccad006657 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -72,6 +72,8 @@ void tqClose(STQ* pTq) { } int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { + // if waiting + // memcpy and send msg to fetch thread // TODO: add reference // if handle waiting, launch query and response to consumer // @@ -210,7 +212,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; int64_t fetchOffset; - /*int64_t blockingTime = pReq->blockingTime;*/ + int64_t blockingTime = pReq->blockingTime; if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { fetchOffset = 0; diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 6d5085ee74..ce00c98ff9 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -90,7 +90,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F char name[pathLen + 10]; strcpy(name, path); - if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { + if (!taosDirExist(name) && taosMkDir(name) != 0) { terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR; tqError("failed to create dir:%s since %s ", name, terrstr()); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 2e7116969e..37403f1a11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -580,7 +580,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pRSmadF = TSDB_READ_SMAD_FILE(&(pCommith->readh)); SDFile *pWSmadF = TSDB_COMMIT_SMAD_FILE(pCommith); - if (access(TSDB_FILE_FULL_NAME(pRSmadF), F_OK) != 0) { + if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmadF))) { tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF)); tsdbInitDFile(pRepo, pWSmadF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAD); @@ -614,7 +614,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pRSmalF = TSDB_READ_SMAL_FILE(&(pCommith->readh)); SDFile *pWSmalF = TSDB_COMMIT_SMAL_FILE(pCommith); - if ((pCommith->isLFileSame) && access(TSDB_FILE_FULL_NAME(pRSmalF), F_OK) == 0) { + if ((pCommith->isLFileSame) && taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmalF))) { tsdbInitDFileEx(pWSmalF, pRSmalF); if (tsdbOpenDFile(pWSmalF, O_RDWR) < 0) { tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), @@ -1054,7 +1054,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // pfs->metaCacheComp = NULL; // } else { // // remove meta.tmp file -// remove(mf.f.aname); +// taosRemoveFile(mf.f.aname); // taosHashCleanup(pfs->metaCacheComp); // pfs->metaCacheComp = NULL; // } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index dcc9700d83..a03739c90f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -314,7 +314,7 @@ int tsdbOpenFS(STsdb *pRepo) { tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); tsdbGetRtnSnap(pRepo, &pRepo->rtn); - if (access(current, F_OK) == 0) { + if (taosCheckExistFile(current)) { if (tsdbOpenFSFromCurrent(pRepo) < 0) { tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; @@ -439,7 +439,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - remove(tfname); + taosRemoveFile(tfname); return -1; } @@ -447,7 +447,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (fsheader.len > 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { taosCloseFile(&pFile); - remove(tfname); + taosRemoveFile(tfname); return -1; } @@ -458,7 +458,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - (void)remove(tfname); + (void)taosRemoveFile(tfname); taosTZfree(pBuf); return -1; } @@ -468,7 +468,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { if (taosFsyncFile(pFile) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(&pFile); - remove(tfname); + taosRemoveFile(tfname); taosTZfree(pBuf); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 4d70c429ff..74fb8c1c1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -443,25 +443,24 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { } static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { - struct stat dfstat; SDFile df; tsdbInitDFileEx(&df, pDFile); - if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) { + if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pDFile))) { tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); // pRepo->state |= TSDB_STATE_BAD_DATA; TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); return 0; } - - if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) { + int64_t file_size = 0; + if (taosStatFile(TSDB_FILE_FULL_NAME(&df), &file_size, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (pDFile->info.size < dfstat.st_size) { + if (pDFile->info.size < file_size) { // if (tsdbOpenDFile(&df, O_WRONLY) < 0) { if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) { return -1; @@ -480,10 +479,10 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { tsdbCloseDFile(&df); tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), - dfstat.st_size, pDFile->info.size); - } else if (pDFile->info.size > dfstat.st_size) { + file_size, pDFile->info.size); + } else if (pDFile->info.size > file_size) { tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", - REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size); + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), file_size, pDFile->info.size); // pRepo->state |= TSDB_STATE_BAD_DATA; TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 78625d1eed..d766f3cbf1 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,4 +1,3 @@ -add_definitions("-D ALLOW_FORBID_FUNC") add_subdirectory(transport) add_subdirectory(sync) add_subdirectory(tdb) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 5779906761..56c0ec1130 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -168,7 +168,7 @@ void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); } - + int32_t colNum = c->numOfColumns + c->numOfTags; for (int32_t i = 0; i < colNum; ++i) { SSchema *s = &p->schema[i]; @@ -190,7 +190,7 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { dbCache = (SCtgDBCache *)pIter; - taosHashGetKey(dbCache, (void **)&dbFName, &len); + taosHashGetKey((void **)&dbFName, &len); int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; @@ -204,9 +204,9 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { if (dbCache->vgInfo->vgHash) { vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); } - } + } - ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", + ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); pIter = taosHashIterate(dbHash, pIter); @@ -222,7 +222,7 @@ void ctgDbgShowClusterCache(SCatalog* pCtg) { } ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg); - ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), + ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); ctgDbgShowDBCache(pCtg, pCtg->dbCache); @@ -306,9 +306,9 @@ int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) return TSDB_CODE_SUCCESS; _return: - + tfree(action.data); - CTG_RET(code); + CTG_RET(code); } @@ -336,9 +336,9 @@ int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId return TSDB_CODE_SUCCESS; _return: - + tfree(action.data); - CTG_RET(code); + CTG_RET(code); } @@ -366,9 +366,9 @@ int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId return TSDB_CODE_SUCCESS; _return: - + tfree(action.data); - CTG_RET(code); + CTG_RET(code); } @@ -657,9 +657,9 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable return TSDB_CODE_SUCCESS; } - size_t sz = 0; + int32_t sz = 0; CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGetCloneExt(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), NULL, (void **)pTableMeta, &sz); + int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), (void **)pTableMeta, &sz); CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == *pTableMeta) { @@ -673,8 +673,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if (dbId) { *dbId = dbCache->dbId; } - - tbMeta = *pTableMeta; + + STableMeta* tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); @@ -1076,7 +1076,7 @@ _return: } int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t compare) { - int16_t widx = abs(id % mgmt->slotNum); + int16_t widx = labs(id % mgmt->slotNum); SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; @@ -1238,7 +1238,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { void *pIter = taosHashIterate(cache->stbCache, NULL); while (pIter) { uint64_t *suid = NULL; - taosHashGetKey(pIter, (void **)&suid, NULL); + suid = taosHashGetKey(pIter, NULL); if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionCompare)) { ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); @@ -1397,7 +1397,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare); } @@ -1437,7 +1437,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, meta->suid); + ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1475,7 +1475,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { int32_t *vgId = NULL; void *pIter = taosHashIterate(src->vgHash, NULL); while (pIter) { - taosHashGetKey(pIter, (void **)&vgId, NULL); + vgId = taosHashGetKey(pIter, NULL); if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) { qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize); @@ -1635,7 +1635,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); } - + if (0 == exist) { CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput)); @@ -1723,9 +1723,9 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons tbType = (*pTableMeta)->tableType; suid = (*pTableMeta)->suid; - tfree(*pTableMeta); + tfree(*pTableMeta); } - + if (CTG_FLAG_IS_UNKNOWN_STB(flag)) { CTG_FLAG_SET_STB(flag, tbType); } @@ -1950,21 +1950,21 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); return TSDB_CODE_SUCCESS; } - + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { + if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } + } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - + ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); - + _return: tfree(msg); - + CTG_RET(code); } @@ -2458,7 +2458,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList)); } else { int32_t vgId = tbMeta->vgId; - if (NULL == taosHashGetClone(vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { + if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) { ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index b417a645be..aeac6b3eee 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -14,9 +14,7 @@ */ #include -#include #include -#include "os.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -24,16 +22,18 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" #pragma GCC diagnostic ignored "-Wformat" +#include -#include "addr_any.h" +#include "os.h" +#include "tglobal.h" #include "catalog.h" +#include "catalogInt.h" #include "stub.h" #include "taos.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "trpc.h" #include "tvariant.h" -#include "catalogInt.h" namespace { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index cff7546a7e..17f457e991 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -68,7 +68,7 @@ typedef struct SResultRow { } SResultRow; typedef struct SResultRowInfo { - SResultRow *pCurResult; // current active result row info + SList* pRows; SResultRow** pResult; // result list // int16_t type:8; // data type for hash key int32_t size; // number of result set diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c8e1aeba0e..991cd372c3 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -15,12 +15,12 @@ #ifndef TDENGINE_EXECUTORIMPL_H #define TDENGINE_EXECUTORIMPL_H -#include "tsort.h" #ifdef __cplusplus extern "C" { #endif #include "os.h" +#include "tsort.h" #include "tcommon.h" #include "tlosertree.h" #include "ttszip.h" @@ -157,6 +157,13 @@ typedef struct STaskCostInfo { SHashObj* operatorProfResults; // map } STaskCostInfo; +typedef struct SOperatorCostInfo { + uint64_t openCost; + uint64_t execCost; + uint64_t totalRows; + uint64_t totalBytes; +} SOperatorCostInfo; + typedef struct { int64_t vgroupLimit; int64_t ts; @@ -233,9 +240,9 @@ typedef struct STaskAttr { SArray* pUdfInfo; // no need to free } STaskAttr; -typedef int32_t (*__optr_prepare_fn_t)(void* param); -typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); -typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); +typedef int32_t (*__optr_open_fn_t)(void* param); +typedef SSDataBlock* (*__optr_fn_t)(void* param, bool* newgroup); +typedef void (*__optr_close_fn_t)(void* param, int32_t num); struct SOperatorInfo; @@ -306,21 +313,21 @@ enum { }; typedef struct SOperatorInfo { - uint8_t operatorType; - bool blockingOptr; // block operator or not - uint8_t status; // denote if current operator is completed - int32_t numOfOutput; // number of columns of the current operator results - char* name; // name, used to show the query execution plan - void* info; // extension attribution - SExprInfo* pExpr; - STaskRuntimeEnv* pRuntimeEnv; // todo remove it - SExecTaskInfo* pTaskInfo; + uint8_t operatorType; + bool blockingOptr; // block operator or not + uint8_t status; // denote if current operator is completed + int32_t numOfOutput; // number of columns of the current operator results + char* name; // name, used to show the query execution plan + void* info; // extension attribution + SExprInfo* pExpr; + STaskRuntimeEnv* pRuntimeEnv; // todo remove it + SExecTaskInfo* pTaskInfo; struct SOperatorInfo** pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - __optr_prepare_fn_t prepareFn; - __operator_fn_t exec; - __optr_cleanup_fn_t cleanupFn; + __optr_open_fn_t openFn; + __optr_fn_t nextDataFn; + __optr_close_fn_t closeFn; } SOperatorInfo; typedef struct { @@ -479,9 +486,6 @@ typedef struct SAggOperatorInfo { typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; - int32_t bufCapacity; - uint32_t seed; - SSDataBlock* existDataBlock; } SProjectOperatorInfo; @@ -615,10 +619,10 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); -SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); @@ -654,8 +658,6 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOf SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); -// SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); -// SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); // SSDataBlock* doSLimit(void* param, bool* newgroup); // int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); diff --git a/source/libs/executor/inc/tsimplehash.h b/source/libs/executor/inc/tsimplehash.h new file mode 100644 index 0000000000..a1ba70c702 --- /dev/null +++ b/source/libs/executor/inc/tsimplehash.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSIMPLEHASH_H +#define TDENGINE_TSIMPLEHASH_H + +#include "tarray.h" +#include "tlockfree.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef uint32_t (*_hash_fn_t)(const char *, uint32_t); +typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); +typedef void (*_hash_free_fn_t)(void *); + +typedef struct SSHashObj SSHashObj; + +/** + * init the hash table + * + * @param capacity initial capacity of the hash table + * @param fn hash function to generate the hash value + * @return + */ +SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen); + +/** + * return the size of hash table + * @param pHashObj + * @return + */ +int32_t tSimpleHashGetSize(const SSHashObj *pHashObj); + +/** + * put element into hash table, if the element with the same key exists, update it + * @param pHashObj + * @param key + * @param data + * @return + */ +int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data); + +/** + * return the payload data with the specified key + * + * @param pHashObj + * @param key + * @return + */ +void *tSimpleHashGet(SSHashObj *pHashObj, const void *key); + +/** + * remove item with the specified key + * @param pHashObj + * @param key + * @param keyLen + */ +int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key); + +/** + * Clear the hash table. + * @param pHashObj + */ +void tSimpleHashClear(SSHashObj *pHashObj); + +/** + * Clean up hash table and release all allocated resources. + * @param handle + */ +void tSimpleHashCleanup(SSHashObj *pHashObj); + +/** + * Get the hash table size + * @param pHashObj + * @return + */ +size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj); + +/** + * Get the corresponding key information for a given data in hash table + * @param data + * @param keyLen + * @return + */ +void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen); + +#ifdef __cplusplus +} +#endif +#endif // TDENGINE_TSIMPLEHASH_H diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b90ce275d2..19a3874f75 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -95,17 +95,17 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { } int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo* )tinfo; + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; // traverse to the streamscan node to add this table id SOperatorInfo* pInfo = pTaskInfo->pRoot; - while(pInfo->operatorType != OP_StreamScan) { + while (pInfo->operatorType != OP_StreamScan) { pInfo = pInfo->pDownstream[0]; } SStreamBlockScanInfo* pScanInfo = pInfo->info; if (isAdd) { - int32_t code = tqReadHandleSetTbUidList(pScanInfo->readerHandle, tableIdList); + int32_t code = tqReadHandleAddTbUidList(pScanInfo->readerHandle, tableIdList); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -114,4 +114,4 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA } return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 1642120d15..fabaa2d31d 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -158,7 +158,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { int64_t st = 0; st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); + *pRes = pTaskInfo->pRoot->nextDataFn(pTaskInfo->pRoot, &newgroup); uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; @@ -179,13 +179,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { return pTaskInfo->code; } -void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - assert(pQInfo != NULL); - - return pQInfo->rspContext; -} - int32_t qKillTask(qTaskInfo_t qinfo) { SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; @@ -221,7 +214,7 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) { int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; - if (pTaskInfo == NULL /*|| !isValidQInfo(pTaskInfo)*/) { + if (pTaskInfo == NULL) { return TSDB_CODE_QRY_INVALID_QHANDLE; } @@ -235,33 +228,3 @@ void qDestroyTask(qTaskInfo_t qTaskHandle) { queryCostStatis(pTaskInfo); // print the query cost summary doDestroyTask(pTaskInfo); } - -#if 0 -//kill by qid -int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) { - int32_t error = TSDB_CODE_SUCCESS; - void** handle = qAcquireTask(pMgmt, qId); - if(handle == NULL) return terrno; - - SQInfo* pQInfo = (SQInfo*)(*handle); - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - qWarn("%s be killed(no memory commit).", pQInfo->qId); - setTaskKilled(pQInfo); - - // wait query stop - int32_t loop = 0; - while (pQInfo->owner != 0) { - taosMsleep(waitMs); - if(loop++ > waitCount){ - error = TSDB_CODE_FAILED; - break; - } - } - - qReleaseTask(pMgmt, (void **)&handle, true); - return error; -} - -#endif diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c400ded1c1..3ebad151fd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -15,12 +15,12 @@ #include "os.h" -#include "tep.h" -#include "tsort.h" -#include "texception.h" #include "parser.h" +#include "tdatablock.h" +#include "texception.h" #include "tglobal.h" #include "tmsg.h" +#include "tsort.h" #include "ttime.h" #include "executorimpl.h" @@ -2215,10 +2215,10 @@ static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) { if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) { SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); if (pColInfoData) { - FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor - if (f) { - fclose(f); - *(FILE **)pColInfoData->pData = NULL; + TdFilePtr pFile = *(TdFilePtr *)pColInfoData->pData; // TODO refactor + if (pFile != NULL) { + taosCloseFile(&pFile); + *(TdFilePtr *)pColInfoData->pData = NULL; } } } @@ -4619,7 +4619,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); int32_t TENMB = 1024*1024*10; - int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, "/tmp"); + int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, "", "/tmp"); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5281,7 +5281,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = size; - pOperator->exec = doLoadRemoteData; + pOperator->nextDataFn = doLoadRemoteData; pOperator->pTaskInfo = pTaskInfo; #if 1 @@ -5361,7 +5361,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->exec = doTableScan; + pOperator->nextDataFn = doTableScan; pOperator->pTaskInfo = pTaskInfo; return pOperator; @@ -5386,7 +5386,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim pOperator->info = pInfo; pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doTableScanImpl; + pOperator->nextDataFn = doTableScanImpl; return pOperator; } @@ -5410,7 +5410,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; // pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->exec = doBlockInfoScan; + pOperator->nextDataFn = doBlockInfoScan; return pOperator; } @@ -5452,7 +5452,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->exec = doStreamBlockScan; + pOperator->nextDataFn = doStreamBlockScan; pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -5663,7 +5663,7 @@ SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*) param; bool newgroup = false; - return pOperator->exec(pOperator, &newgroup); + return pOperator->nextDataFn(pOperator, &newgroup); } static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char **buf, int32_t rowIndex) { @@ -5983,8 +5983,8 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pOperator->pExpr = exprArrayDup(pExprInfo); pOperator->pTaskInfo = pTaskInfo; - pOperator->exec = doSortedMerge; - pOperator->cleanupFn = destroySortedMergeOperatorInfo; + pOperator->nextDataFn = doSortedMerge; + pOperator->closeFn = destroySortedMergeOperatorInfo; code = appendDownstream(pOperator, downstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -6079,8 +6079,8 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->exec = doSort; - pOperator->cleanupFn = destroyOrderOperatorInfo; + pOperator->nextDataFn = doSort; + pOperator->closeFn = destroyOrderOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -6105,7 +6105,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6155,7 +6155,7 @@ static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6224,7 +6224,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows); projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); @@ -6241,7 +6241,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // The downstream exec may change the value of the newgroup, so use a local variable instead. publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6274,7 +6274,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows); projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); @@ -6299,7 +6299,7 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) { SSDataBlock* pBlock = NULL; while (1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6350,7 +6350,7 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) { while (1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock *pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + SSDataBlock *pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6393,7 +6393,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6453,7 +6453,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6516,7 +6516,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6571,7 +6571,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6706,7 +6706,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6768,7 +6768,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -6821,7 +6821,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -6906,7 +6906,7 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { @@ -6979,8 +6979,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { return; } - if (pOperator->cleanupFn != NULL) { - pOperator->cleanupFn(pOperator->info, pOperator->numOfOutput); + if (pOperator->closeFn != NULL) { + pOperator->closeFn(pOperator->info, pOperator->numOfOutput); } if (pOperator->pDownstream != NULL) { @@ -7067,8 +7067,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE pOperator->numOfOutput = taosArrayGetSize(pExprInfo); pOperator->pTaskInfo = pTaskInfo; - pOperator->exec = doAggregate; - pOperator->cleanupFn = destroyAggOperatorInfo; + pOperator->nextDataFn = doAggregate; + pOperator->closeFn = destroyAggOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7164,38 +7164,34 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray pOperator->pExpr = exprArrayDup(pExprInfo); pOperator->numOfOutput = numOfOutput; - pOperator->exec = doMultiTableAggregate; - pOperator->cleanupFn = destroyAggOperatorInfo; + pOperator->nextDataFn = doMultiTableAggregate; + pOperator->closeFn = destroyAggOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } -SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); - pInfo->seed = rand(); - pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; + int32_t numOfRows = 4096; + pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); - SOptrBasicInfo* pBInfo = &pInfo->binfo; - pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); - pBInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); - - initResultRowInfo(&pBInfo->resultRowInfo, 8); - setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN); +// initResultRowInfo(&pBInfo->resultRowInfo, 8); +// setDefaultOutputBuf_rv(pBInfo, MAIN_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "ProjectOperator"; -// pOperator->operatorType = OP_Project; + pOperator->operatorType = OP_Project; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->pExpr = exprArrayDup(pExprInfo); + pOperator->numOfOutput = taosArrayGetSize(pExprInfo); - pOperator->exec = doProjectOperation; - pOperator->cleanupFn = destroyProjectOperatorInfo; + pOperator->nextDataFn = doProjectOperation; + pOperator->closeFn = destroyProjectOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7250,10 +7246,10 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->status = OP_IN_EXECUTING; pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; - pOperator->exec = doFilter; + pOperator->nextDataFn = doFilter; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanupFn = destroyConditionOperatorInfo; + pOperator->closeFn = destroyConditionOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7269,7 +7265,7 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn // pOperator->operatorType = OP_Limit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doLimit; + pOperator->nextDataFn = doLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -7277,24 +7273,18 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn return pOperator; } -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); initAggSup(&pInfo->aggSup, pExprInfo); - // todo: pInfo->order = TSDB_ORDER_ASC; pInfo->precision = TSDB_TIME_PRECISION_MICRO; - pInfo->win.skey = INT64_MIN; - pInfo->win.ekey = INT64_MAX; - pInfo->interval.intervalUnit = 's'; - pInfo->interval.slidingUnit = 's'; - pInfo->interval.interval = 1000; - pInfo->interval.sliding = 1000; + pInfo->win = pTaskInfo->window; + pInfo->interval = *pInterval; - int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, 0, "/tmp/"); + int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); - int32_t numOfOutput = taosArrayGetSize(pExprInfo); pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); @@ -7309,16 +7299,15 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pEx pOperator->pExpr = exprArrayDup(pExprInfo); pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfOutput; + pOperator->numOfOutput = taosArrayGetSize(pExprInfo); pOperator->info = pInfo; - pOperator->exec = doIntervalAgg; - pOperator->cleanupFn = destroyBasicOperatorInfo; + pOperator->nextDataFn = doIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; code = appendDownstream(pOperator, &downstream, 1); return pOperator; } - SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); @@ -7336,8 +7325,8 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doAllIntervalAgg; - pOperator->cleanupFn = destroyBasicOperatorInfo; + pOperator->nextDataFn = doAllIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7360,8 +7349,8 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doStateWindowAgg; - pOperator->cleanupFn = destroyStateWindowOperatorInfo; + pOperator->nextDataFn = doStateWindowAgg; + pOperator->closeFn = destroyStateWindowOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7385,8 +7374,8 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doSessionWindowAgg; - pOperator->cleanupFn = destroySWindowOperatorInfo; + pOperator->nextDataFn = doSessionWindowAgg; + pOperator->closeFn = destroySWindowOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7409,8 +7398,8 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doSTableIntervalAgg; - pOperator->cleanupFn = destroyBasicOperatorInfo; + pOperator->nextDataFn = doSTableIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7433,8 +7422,8 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doAllSTableIntervalAgg; - pOperator->cleanupFn = destroyBasicOperatorInfo; + pOperator->nextDataFn = doAllSTableIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -7465,8 +7454,8 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashGroupbyAggregate; - pOperator->cleanupFn = destroyGroupbyOperatorInfo; + pOperator->nextDataFn = hashGroupbyAggregate; + pOperator->closeFn = destroyGroupbyOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7504,8 +7493,8 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doFill; - pOperator->cleanupFn = destroySFillOperatorInfo; + pOperator->nextDataFn = doFill; + pOperator->closeFn = destroySFillOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7553,7 +7542,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI // pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanupFn = destroySlimitOperatorInfo; + pOperator->closeFn = destroySlimitOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7707,11 +7696,11 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->exec = doTagScan; + pOperator->nextDataFn = doTagScan; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanupFn = destroyTagScanOperatorInfo; + pOperator->closeFn = destroyTagScanOperatorInfo; return pOperator; } @@ -7777,7 +7766,7 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { while(1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup); publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -7849,9 +7838,9 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = hashDistinct; + pOperator->nextDataFn = hashDistinct; pOperator->pExpr = pExpr; - pOperator->cleanupFn = destroyDistinctOperatorInfo; + pOperator->closeFn = destroyDistinctOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -8741,10 +8730,8 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type static int64_t getQuerySupportBufSize(size_t numOfTables) { size_t s1 = sizeof(STableQueryInfo); - size_t s2 = sizeof(SHashNode); - // size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb - return (int64_t)((s1 + s2) * 1.5 * numOfTables); + return (int64_t)(s1* 1.5 * numOfTables); } int32_t checkForQueryBuf(size_t numOfTables) { diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 7f3da9bcac..9f3d694c42 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -29,8 +29,8 @@ typedef struct SLHashBucket { typedef struct SLHashObj { SDiskbasedBuf *pBuf; _hash_fn_t hashFn; - int32_t tuplesPerPage; SLHashBucket **pBucket; // entry list + int32_t tuplesPerPage; int32_t numOfAlloc; // number of allocated bucket ptr slot int32_t bits; // the number of bits used in hash int32_t numOfBuckets; // the number of buckets @@ -142,7 +142,7 @@ static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket pBucket->size -= 1; } -static void doCompressBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) { +static void doTrimBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) { size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList); if (numOfPages <= 1) { return; @@ -253,6 +253,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ return NULL; } + // disable compress when flushing to disk setBufPageCompressOnDisk(pHashObj->pBuf, false); /** @@ -367,7 +368,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data releaseBufPage(pHashObj->pBuf, p); } - doCompressBucketPages(pHashObj, pBucket); + doTrimBucketPages(pHashObj, pBucket); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c new file mode 100644 index 0000000000..4012c3926e --- /dev/null +++ b/source/libs/executor/src/tsimplehash.c @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tsimplehash.h" +#include "taoserror.h" + +#define SHASH_DEFAULT_LOAD_FACTOR 0.75 +#define HASH_MAX_CAPACITY (1024*1024*16) +#define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) + +#define GET_SHASH_NODE_KEY(_n, _dl) ((char*)(_n) + sizeof(SHNode) + (_dl)) +#define GET_SHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHNode)) + +#define HASH_INDEX(v, c) ((v) & ((c)-1)) +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) + +#define FREE_HASH_NODE(_n) \ + do { \ + tfree(_n); \ + } while (0); + +typedef struct SHNode { + struct SHNode *next; + char data[]; +} SHNode; + +typedef struct SSHashObj { + SHNode **hashList; + size_t capacity; // number of slots + size_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + int32_t keyLen; + int32_t dataLen; +} SSHashObj; + +static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { + int32_t len = MIN(length, HASH_MAX_CAPACITY); + + int32_t i = 4; + while (i < len) i = (i << 1u); + return i; +} + +SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen) { + ASSERT(fn != NULL); + + if (capacity == 0) { + capacity = 4; + } + + SSHashObj* pHashObj = (SSHashObj*) calloc(1, sizeof(SSHashObj)); + if (pHashObj == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + // the max slots is not defined by user + pHashObj->capacity = taosHashCapacity((int32_t)capacity); + + pHashObj->equalFp = memcmp; + pHashObj->hashFp = fn; + ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); + + pHashObj->keyLen = keyLen; + pHashObj->dataLen = dataLen; + + pHashObj->hashList = (SHNode **)calloc(pHashObj->capacity, sizeof(void *)); + if (pHashObj->hashList == NULL) { + free(pHashObj); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + return pHashObj; +} + +int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + return (int32_t)atomic_load_64(&pHashObj->size); +} + +static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { + SHNode *pNewNode = malloc(sizeof(SHNode) + keyLen + dsize); + if (pNewNode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pNewNode->next = NULL; + memcpy(GET_SHASH_NODE_DATA(pNewNode), pData, dsize); + memcpy(GET_SHASH_NODE_KEY(pNewNode, dsize), key, keyLen); + return pNewNode; +} + +void taosHashTableResize(SSHashObj *pHashObj) { + if (!HASH_NEED_RESIZE(pHashObj)) { + return; + } + + int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); + if (newCapacity > HASH_MAX_CAPACITY) { +// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", +// pHashObj->capacity, HASH_MAX_CAPACITY); + return; + } + + int64_t st = taosGetTimestampUs(); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + if (pNewEntryList == NULL) { +// qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); + return; + } + + size_t inc = newCapacity - pHashObj->capacity; + memset(pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); + + pHashObj->hashList = pNewEntryList; + pHashObj->capacity = newCapacity; + + for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { + SHNode* pNode = pHashObj->hashList[idx]; + SHNode *pNext; + SHNode *pPrev = NULL; + + if (pNode == NULL) { + continue; + } + + while (pNode != NULL) { + void* key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen); + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->dataLen); + + int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity); + pNext = pNode->next; + if (newIdx != idx) { + if (pPrev == NULL) { + pHashObj->hashList[idx] = pNext; + } else { + pPrev->next = pNext; + } + + pNode->next = pHashObj->hashList[newIdx]; + pHashObj->hashList[newIdx] = pNode; + } else { + pPrev = pNode; + } + + pNode = pNext; + } + } + + int64_t et = taosGetTimestampUs(); + +// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, +// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); +} + +int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { + if (pHashObj == NULL || key == NULL) { + return -1; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen); + + // need the resize process, write lock applied + if (SHASH_NEED_RESIZE(pHashObj)) { + taosHashTableResize(pHashObj); + } + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + + SHNode *pNode = pHashObj->hashList[slot]; + if (pNode == NULL) { + SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal); + if (pNewNode == NULL) { + return -1; + } + + pHashObj->hashList[slot] = pNewNode; + return 0; + } + + while (pNode) { + if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) { + break; + } + pNode = pNode->next; + } + + if (pNode == NULL) { + SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal); + if (pNewNode == NULL) { + return -1; + } + pNewNode->next = pHashObj->hashList[slot]; + pHashObj->hashList[slot] = pNewNode; + atomic_add_fetch_64(&pHashObj->size, 1); + } else { //update data + memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen); + } + + return 0; +} + +static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, int32_t index) { + SHNode *pNode = pHashObj->hashList[index]; + while (pNode) { + if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) { + break; + } + + pNode = pNode->next; + } + + return pNode; +} + +static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { + return tSimpleHashGetSize(pHashObj) == 0; +} + +void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL) { + return NULL; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHNode *pNode = pHashObj->hashList[slot]; + if (pNode == NULL) { + return NULL; + } + + char *data = NULL; + pNode = doSearchInEntryList(pHashObj, key, slot); + if (pNode != NULL) { + data = GET_SHASH_NODE_DATA(pNode); + } + + return data; +} + +int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key) { + // todo +} + +void tSimpleHashClear(SSHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + SHNode *pNode, *pNext; + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + pNode = pHashObj->hashList[i]; + if (pNode == NULL) { + continue; + } + + while (pNode) { + pNext = pNode->next; + FREE_HASH_NODE(pNode); + pNode = pNext; + } + } + pHashObj->size = 0; +} + +void tSimpleHashCleanup(SSHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + tSimpleHashClear(pHashObj); + tfree(pHashObj->hashList); +} + +size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + + return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj); +} + +void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) { + int32_t offset = offsetof(SHNode, data); + SHNode *node = data - offset; + if (keyLen != NULL) { + *keyLen = pHashObj->keyLen; + } + + return GET_SHASH_NODE_KEY(node, pHashObj->dataLen); +} \ No newline at end of file diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index d042dc0eff..34dd248ba7 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -16,11 +16,11 @@ #include "tcommon.h" #include "query.h" -#include "tsort.h" -#include "tep.h" +#include "tdatablock.h" #include "tdef.h" #include "tlosertree.h" #include "tpagedbuf.h" +#include "tsort.h" #include "tutil.h" typedef struct STupleHandle { diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 262232ceb1..ff29d5f355 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -13,10 +13,7 @@ * along with this program. If not, see . */ -#include -#include #include -#include #include #pragma GCC diagnostic push @@ -26,10 +23,13 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "tglobal.h" +#include "executorimpl.h" +#include "function.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" -#include "tep.h" +#include "tdatablock.h" #include "trpc.h" #include "stub.h" #include "executor.h" @@ -201,9 +201,9 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_ pOperator->name = "dummyInputOpertor4Test"; if (numOfCols == 1) { - pOperator->exec = getDummyBlock; + pOperator->nextDataFn = getDummyBlock; } else { - pOperator->exec = get2ColsDummyBlock; + pOperator->nextDataFn = get2ColsDummyBlock; } SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); @@ -222,100 +222,600 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } -#if 0 TEST(testCase, build_executor_tree_Test) { const char* msg = "{\n" - "\t\"Id\":\t{\n" - "\t\t\"QueryId\":\t1.3108161807422521e+19,\n" - "\t\t\"TemplateId\":\t0,\n" - "\t\t\"SubplanId\":\t0\n" - "\t},\n" - "\t\"Node\":\t{\n" - "\t\t\"Name\":\t\"TableScan\",\n" - "\t\t\"Targets\":\t[{\n" - "\t\t\t\t\"Base\":\t{\n" - "\t\t\t\t\t\"Schema\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\t\t\"ColId\":\t5000,\n" - "\t\t\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t\t\t},\n" - "\t\t\t\t\t\"Columns\":\t[{\n" - "\t\t\t\t\t\t\t\"TableId\":\t1,\n" - "\t\t\t\t\t\t\t\"Flag\":\t0,\n" - "\t\t\t\t\t\t\t\"Info\":\t{\n" - "\t\t\t\t\t\t\t\t\"ColId\":\t1,\n" - "\t\t\t\t\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t\t\t\t\t}\n" - "\t\t\t\t\t\t}],\n" - "\t\t\t\t\t\"InterBytes\":\t0\n" - "\t\t\t\t},\n" - "\t\t\t\t\"Expr\":\t{\n" - "\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\"Column\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\t\t\"ColId\":\t1,\n" - "\t\t\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t\t\t}\n" - "\t\t\t\t}\n" - "\t\t\t}, {\n" - "\t\t\t\t\"Base\":\t{\n" - "\t\t\t\t\t\"Schema\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\t\"ColId\":\t5001,\n" - "\t\t\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t\t\t},\n" - "\t\t\t\t\t\"Columns\":\t[{\n" - "\t\t\t\t\t\t\t\"TableId\":\t1,\n" - "\t\t\t\t\t\t\t\"Flag\":\t0,\n" - "\t\t\t\t\t\t\t\"Info\":\t{\n" - "\t\t\t\t\t\t\t\t\"ColId\":\t2,\n" - "\t\t\t\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t\t\t\t\t}\n" - "\t\t\t\t\t\t}],\n" - "\t\t\t\t\t\"InterBytes\":\t0\n" - "\t\t\t\t},\n" - "\t\t\t\t\"Expr\":\t{\n" - "\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\"Column\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\t\"ColId\":\t2,\n" - "\t\t\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t\t\t}\n" - "\t\t\t\t}\n" - "\t\t\t}],\n" - "\t\t\"InputSchema\":\t[{\n" - "\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\"ColId\":\t5000,\n" - "\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t}, {\n" - "\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\"ColId\":\t5001,\n" - "\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t}],\n" - "\t\t\"TableScan\":\t{\n" - "\t\t\t\"TableId\":\t1,\n" - "\t\t\t\"TableType\":\t2,\n" - "\t\t\t\"Flag\":\t0,\n" - "\t\t\t\"Window\":\t{\n" - "\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n" - "\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n" - "\t\t\t}\n" - "\t\t}\n" - "\t},\n" - "\t\"DataSink\":\t{\n" - "\t\t\"Name\":\t\"Dispatch\",\n" - "\t\t\"Dispatch\":\t{\n" - "\t\t}\n" - "\t}\n" + " \"Type\": \"33\",\n" + " \"Name\": \"PhysiProject\",\n" + " \"PhysiProject\": {\n" + " \"OutputDataBlockDesc\": {\n" + " \"Type\": \"19\",\n" + " \"Name\": \"TupleDesc\",\n" + " \"TupleDesc\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"Slots\": [\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"0\",\n" + " \"DataType\": {\n" + " \"Type\": \"9\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"1\",\n" + " \"DataType\": {\n" + " \"Type\": \"4\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"4\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"2\",\n" + " \"DataType\": {\n" + " \"Type\": \"8\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"20\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"3\",\n" + " \"DataType\": {\n" + " \"Type\": \"5\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"4\",\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"5\",\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": false\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " },\n" + " \"Children\": [\n" + " {\n" + " \"Type\": \"30\",\n" + " \"Name\": \"PhysiTableScan\",\n" + " \"PhysiTableScan\": {\n" + " \"OutputDataBlockDesc\": {\n" + " \"Type\": \"19\",\n" + " \"Name\": \"TupleDesc\",\n" + " \"TupleDesc\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"Slots\": [\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"0\",\n" + " \"DataType\": {\n" + " \"Type\": \"9\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"1\",\n" + " \"DataType\": {\n" + " \"Type\": \"4\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"4\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"2\",\n" + " \"DataType\": {\n" + " \"Type\": \"8\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"20\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"3\",\n" + " \"DataType\": {\n" + " \"Type\": \"5\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"4\",\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"20\",\n" + " \"Name\": \"SlotDesc\",\n" + " \"SlotDesc\": {\n" + " \"SlotId\": \"5\",\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"Reserve\": false,\n" + " \"Output\": true\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " },\n" + " \"ScanCols\": [\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"9\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"ts\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"1\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"ts\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"1\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"4\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"4\"\n" + " },\n" + " \"AliasName\": \"c1\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"2\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c1\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"2\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"8\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"20\"\n" + " },\n" + " \"AliasName\": \"c2\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"3\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c2\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"3\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"5\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c3\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"4\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c3\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"4\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c4\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"5\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c4\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"5\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c5\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"6\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c5\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " }\n" + " ],\n" + " \"TableId\": \"1\",\n" + " \"TableType\": \"3\",\n" + " \"ScanOrder\": \"1\",\n" + " \"ScanCount\": \"1\",\n" + " \"ReverseScanCount\": \"0\",\n" + " \"ScanFlag\": \"0\",\n" + " \"StartKey\": \"-9223372036854775808\",\n" + " \"EndKey\": \"9223372036854775807\"\n" + " }\n" + " }\n" + " ],\n" + " \"Projections\": [\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"0\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"9\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"ts\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"1\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"ts\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"0\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"1\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"4\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"4\"\n" + " },\n" + " \"AliasName\": \"c1\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"2\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c1\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"1\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"2\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"8\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"20\"\n" + " },\n" + " \"AliasName\": \"c2\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"3\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c2\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"2\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"3\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"5\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c3\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"4\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c3\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"3\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"4\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c4\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"5\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c4\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"4\"\n" + " }\n" + " }\n" + " }\n" + " },\n" + " {\n" + " \"Type\": \"18\",\n" + " \"Name\": \"Target\",\n" + " \"Target\": {\n" + " \"DataBlockId\": \"1\",\n" + " \"SlotId\": \"5\",\n" + " \"Expr\": {\n" + " \"Type\": \"1\",\n" + " \"Name\": \"Column\",\n" + " \"Column\": {\n" + " \"DataType\": {\n" + " \"Type\": \"7\",\n" + " \"Precision\": \"0\",\n" + " \"Scale\": \"0\",\n" + " \"Bytes\": \"8\"\n" + " },\n" + " \"AliasName\": \"c5\",\n" + " \"TableId\": \"0\",\n" + " \"ColId\": \"6\",\n" + " \"ColType\": \"1\",\n" + " \"DbName\": \"test\",\n" + " \"TableName\": \"t1\",\n" + " \"TableAlias\": \"t1\",\n" + " \"ColName\": \"c5\",\n" + " \"DataBlockId\": \"0\",\n" + " \"SlotId\": \"5\"\n" + " }\n" + " }\n" + " }\n" + " }\n" + " ]\n" + " }\n" "}"; SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; SReadHandle handle = {.reader = reinterpret_cast(0x1), .meta = reinterpret_cast(0x1)}; -// int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); + SSubplan* plan = NULL; + qStringToSubplan(msg, &plan); + + int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); } TEST(testCase, inMem_sort_Test) { @@ -334,10 +834,10 @@ TEST(testCase, inMem_sort_Test) { exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); taosArrayPush(pExprInfo, &exp1); - SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal, NULL); + SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 5, 1000, data_asc, 1), pExprInfo, pOrderVal, NULL); bool newgroup = false; - SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup); + SSDataBlock* pRes = pOperator->nextDataFn(pOperator, &newgroup); SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); @@ -400,7 +900,7 @@ TEST(testCase, external_sort_Test) { exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); // taosArrayPush(pExprInfo, &exp1); - SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal, NULL); + SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 1500, 1000, data_desc, 1), pExprInfo, pOrderVal, NULL); bool newgroup = false; SSDataBlock* pRes = NULL; @@ -412,7 +912,7 @@ TEST(testCase, external_sort_Test) { while(1) { int64_t s = taosGetTimestampUs(); - pRes = pOperator->exec(pOperator, &newgroup); + pRes = pOperator->nextDataFn(pOperator, &newgroup); int64_t e = taosGetTimestampUs(); if (t++ == 1) { @@ -435,7 +935,7 @@ TEST(testCase, external_sort_Test) { int64_t s2 = taosGetTimestampUs(); printf("total:%ld\n", s2 - s1); - pOperator->cleanupFn(pOperator->info, 2); + pOperator->closeFn(pOperator->info, 2); tfree(exp); tfree(exp1); taosArrayDestroy(pExprInfo); @@ -469,7 +969,7 @@ TEST(testCase, sorted_merge_Test) { int32_t numOfSources = 10; SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*)); for(int32_t i = 0; i < numOfSources; ++i) { - plist[i] = createDummyOperator(1, 1, 1, data_asc); + plist[i] = createDummyOperator(1, 1, 1, data_asc, 1); } SOperatorInfo* pOperator = createSortedMergeOperatorInfo(plist, numOfSources, pExprInfo, pOrderVal, NULL, NULL); @@ -484,7 +984,7 @@ TEST(testCase, sorted_merge_Test) { while(1) { int64_t s = taosGetTimestampUs(); - pRes = pOperator->exec(pOperator, &newgroup); + pRes = pOperator->nextDataFn(pOperator, &newgroup); int64_t e = taosGetTimestampUs(); if (t++ == 1) { @@ -507,15 +1007,13 @@ TEST(testCase, sorted_merge_Test) { int64_t s2 = taosGetTimestampUs(); printf("total:%ld\n", s2 - s1); - pOperator->cleanupFn(pOperator->info, 2); + pOperator->closeFn(pOperator->info, 2); tfree(exp); tfree(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } -#endif - TEST(testCase, time_interval_Operator_Test) { srand(time(NULL)); @@ -548,7 +1046,11 @@ TEST(testCase, time_interval_Operator_Test) { SOperatorInfo* p = createDummyOperator(1, 1, 2000, data_asc, 2); SExecTaskInfo ti = {0}; - SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &ti); + SInterval interval = {0}; + interval.sliding = interval.interval = 1000; + interval.slidingUnit = interval.intervalUnit = 'a'; + + SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &interval, &ti); bool newgroup = false; SSDataBlock* pRes = NULL; @@ -560,7 +1062,7 @@ TEST(testCase, time_interval_Operator_Test) { while(1) { int64_t s = taosGetTimestampUs(); - pRes = pOperator->exec(pOperator, &newgroup); + pRes = pOperator->nextDataFn(pOperator, &newgroup); int64_t e = taosGetTimestampUs(); if (t++ == 1) { @@ -583,7 +1085,7 @@ TEST(testCase, time_interval_Operator_Test) { int64_t s2 = taosGetTimestampUs(); printf("total:%ld\n", s2 - s1); - pOperator->cleanupFn(pOperator->info, 2); + pOperator->closeFn(pOperator->info, 2); tfree(exp); tfree(exp1); taosArrayDestroy(pExprInfo); diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index fc366e4cc8..1956794395 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include #include #include @@ -26,11 +25,12 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "executorimpl.h" #include "executor.h" #include "stub.h" #include "taos.h" +#include "tdatablock.h" #include "tdef.h" -#include "tep.h" #include "trpc.h" #include "tvariant.h" diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index c36fc9c659..7fb63f5910 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -28,8 +28,8 @@ #include "tbuffer.h" #include "tcompression.h" //#include "queryLog.h" +#include "tdatablock.h" #include "tudf.h" -#include "tep.h" #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) #define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y))) diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 1aa0c04ec4..06c58430a4 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -255,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, "/tmp"); + int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", "/tmp"); if (ret != 0) { tMemBucketDestroy(pBucket); return NULL; diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 0fcaf11087..d47954ac1f 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -96,7 +96,7 @@ typedef struct SIndexTermQuery { typedef struct Iterate Iterate; typedef struct IterateValue { - int8_t type; + int8_t type; // opera type, ADD_VALUE/DELETE_VALUE char* colVal; SArray* val; } IterateValue; @@ -125,41 +125,41 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); -#define indexFatal(...) \ - do { \ - if (sDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("index FATAL ", 255, __VA_ARGS__); \ - } \ +#define indexFatal(...) \ + do { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("index FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } while (0) -#define indexError(...) \ - do { \ - if (sDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("index ERROR ", 255, __VA_ARGS__); \ - } \ +#define indexError(...) \ + do { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("index ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } while (0) -#define indexWarn(...) \ - do { \ - if (sDebugFlag & DEBUG_WARN) { \ - taosPrintLog("index WARN ", 255, __VA_ARGS__); \ - } \ +#define indexWarn(...) \ + do { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("index WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } while (0) -#define indexInfo(...) \ - do { \ - if (sDebugFlag & DEBUG_INFO) { \ - taosPrintLog("index ", 255, __VA_ARGS__); \ - } \ +#define indexInfo(...) \ + do { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("index ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } while (0) -#define indexDebug(...) \ - do { \ - if (sDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ - } \ +#define indexDebug(...) \ + do { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("index ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \ + } \ } while (0) -#define indexTrace(...) \ - do { \ - if (sDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ - } \ +#define indexTrace(...) \ + do { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("index ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \ + } \ } while (0) #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index 985dd657d0..313839bf1d 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -54,7 +54,22 @@ extern "C" { * output:[4, 5] */ void iIntersection(SArray *interResults, SArray *finalResult); + +/* multi sorted result intersection + * input: [1, 2, 4, 5] + * [2, 3, 4, 5] + * [1, 4, 5] + * output:[1, 2, 3, 4, 5] + */ void iUnion(SArray *interResults, SArray *finalResult); + +/* sorted array + * total: [1, 2, 4, 5, 7, 8] + * except: [4, 5] + * return: [1, 2, 7, 8] + */ + +void iExcept(SArray *total, SArray *except); #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 168e819073..75dc05fc5b 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -16,6 +16,7 @@ #include "index.h" #include "indexInt.h" #include "index_cache.h" +#include "index_comm.h" #include "index_tfile.h" #include "index_util.h" #include "tdef.h" @@ -30,8 +31,18 @@ void* indexQhandle = NULL; -static char JSON_COLUMN[] = "JSON"; - +#define INDEX_MERGE_ADD_DEL(src, dst, tgt) \ + { \ + bool f = false; \ + for (int i = 0; i < taosArrayGetSize(src); i++) { \ + if (*(uint64_t*)taosArrayGet(src, i) == tgt) { \ + f = true; \ + } \ + } \ + if (f == false) { \ + taosArrayPush(dst, &tgt); \ + } \ + } void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); @@ -52,6 +63,12 @@ typedef struct SIdxColInfo { int cVersion; } SIdxColInfo; +typedef struct SIdxTempResult { + SArray* total; + SArray* added; + SArray* deled; +} SIdxTempResult; + static pthread_once_t isInit = PTHREAD_ONCE_INIT; // static void indexInit(); static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); @@ -62,8 +79,7 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); // merge cache and tfile by opera type -static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv); -static void indexMergeSameKey(SArray* result, TFileValue* tv); +static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv, SIdxTempResult* helper); // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); @@ -379,7 +395,6 @@ static void indexInterResultsDestroy(SArray* results) { static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { // refactor, merge interResults into fResults by oType - for (int i = 0; i < taosArrayGetSize(interResults); i--) { SArray* t = taosArrayGetP(interResults, i); taosArraySort(t, uidCompare); @@ -398,44 +413,82 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType return 0; } -static void indexMergeSameKey(SArray* result, TFileValue* tv) { - int32_t sz = result ? taosArrayGetSize(result) : 0; +SIdxTempResult* sIdxTempResultCreate() { + SIdxTempResult* tr = calloc(1, sizeof(SIdxTempResult)); + tr->total = taosArrayInit(4, sizeof(uint64_t)); + tr->added = taosArrayInit(4, sizeof(uint64_t)); + tr->deled = taosArrayInit(4, sizeof(uint64_t)); + return tr; +} +void sIdxTempResultClear(SIdxTempResult* tr) { + if (tr == NULL) { + return; + } + taosArrayClear(tr->total); + taosArrayClear(tr->added); + taosArrayClear(tr->deled); +} +void sIdxTempResultDestroy(SIdxTempResult* tr) { + if (tr == NULL) { + return; + } + taosArrayDestroy(tr->total); + taosArrayDestroy(tr->added); + taosArrayDestroy(tr->deled); +} +static void sIdxTempResultMergeTo(SArray* result, SIdxTempResult* tr) { + taosArraySort(tr->total, uidCompare); + taosArraySort(tr->added, uidCompare); + taosArraySort(tr->deled, uidCompare); + + SArray* arrs = taosArrayInit(2, sizeof(void*)); + taosArrayPush(arrs, &tr->total); + taosArrayPush(arrs, &tr->added); + + iUnion(arrs, result); + taosArrayDestroy(arrs); + + iExcept(result, tr->deled); +} +static void indexMayMergeTempToFinalResult(SArray* result, TFileValue* tfv, SIdxTempResult* tr) { + int32_t sz = taosArrayGetSize(result); if (sz > 0) { - // TODO(yihao): remove duplicate tableid TFileValue* lv = taosArrayGetP(result, sz - 1); - // indexError("merge colVal: %s", lv->colVal); - if (strcmp(lv->colVal, tv->colVal) == 0) { - taosArrayAddAll(lv->tableId, tv->tableId); - tfileValueDestroy(tv); + if (tfv != NULL && strcmp(lv->colVal, tfv->colVal) != 0) { + sIdxTempResultMergeTo(lv->tableId, tr); + sIdxTempResultClear(tr); + + taosArrayPush(result, &tfv); + } else if (tfv == NULL) { + // handle last iterator + sIdxTempResultMergeTo(lv->tableId, tr); } else { - taosArrayPush(result, &tv); + // temp result saved in help + tfileValueDestroy(tfv); } } else { - taosArrayPush(result, &tv); + taosArrayPush(result, &tfv); } } -static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv) { - // opt - char* colVal = (cv != NULL) ? cv->colVal : tv->colVal; - // design merge-algorithm later, too complicated to handle all kind of situation +static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv, SIdxTempResult* tr) { + char* colVal = (cv != NULL) ? cv->colVal : tv->colVal; TFileValue* tfv = tfileValueCreate(colVal); + + indexMayMergeTempToFinalResult(result, tfv, tr); + if (cv != NULL) { + uint64_t id = *(uint64_t*)taosArrayGet(cv->val, 0); if (cv->type == ADD_VALUE) { - taosArrayAddAll(tfv->tableId, cv->val); + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, id) } else if (cv->type == DEL_VALUE) { - } else if (cv->type == UPDATE_VALUE) { - } else { - // do nothing + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, id) } } if (tv != NULL) { - // opt later - taosArrayAddAll(tfv->tableId, tv->val); + taosArrayAddAll(tr->total, tv->val); } - - indexMergeSameKey(result, tfv); } -static void indexDestroyTempResult(SArray* result) { +static void indexDestroyFinalResult(SArray* result) { int32_t sz = result ? taosArrayGetSize(result) : 0; for (size_t i = 0; i < sz; i++) { TFileValue* tv = taosArrayGetP(result, i); @@ -443,6 +496,7 @@ static void indexDestroyTempResult(SArray* result) { } taosArrayDestroy(result); } + int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { if (sIdx == NULL) { return -1; @@ -467,6 +521,8 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { bool cn = cacheIter ? cacheIter->next(cacheIter) : false; bool tn = tfileIter ? tfileIter->next(tfileIter) : false; + + SIdxTempResult* tr = sIdxTempResultCreate(); while (cn == true || tn == true) { IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL; IterateValue* tv = (tn == true) ? tfileIter->getValue(tfileIter) : NULL; @@ -480,19 +536,22 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { comp = 1; } if (comp == 0) { - indexMergeCacheAndTFile(result, cv, tv); + indexMergeCacheAndTFile(result, cv, tv, tr); cn = cacheIter->next(cacheIter); tn = tfileIter->next(tfileIter); } else if (comp < 0) { - indexMergeCacheAndTFile(result, cv, NULL); + indexMergeCacheAndTFile(result, cv, NULL, tr); cn = cacheIter->next(cacheIter); } else { - indexMergeCacheAndTFile(result, NULL, tv); + indexMergeCacheAndTFile(result, NULL, tv, tr); tn = tfileIter->next(tfileIter); } } + indexMayMergeTempToFinalResult(result, NULL, tr); + sIdxTempResultDestroy(tr); + int ret = indexGenTFile(sIdx, pCache, result); - indexDestroyTempResult(result); + indexDestroyFinalResult(result); indexCacheDestroyImm(pCache); diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 599bac3fe6..1ac72e10a9 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -267,6 +267,10 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA SSkipListNode* node = tSkipListIterGet(iter); if (node != NULL) { CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + // if (c->operaType == ADD_VALUE) { + //} else if (c->operaType == DEL_VALUE) { + //} + if (c->operaType == ADD_VALUE || qtype == QUERY_TERM) { if (strcmp(c->colVal, ct->colVal) == 0) { taosArrayPush(result, &c->uid); @@ -411,17 +415,8 @@ static bool indexCacheIteratorNext(Iterate* itera) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); - // equal func - // if (iv->colVal != NULL && ct->colVal != NULL) { - // if (0 == strcmp(iv->colVal, ct->colVal)) { if (iv->type == ADD_VALUE) } - //} else { - // tIterVal.colVal = calloc(1, strlen(ct->colVal) + 1); - // tIterval.colVal = tstrdup(ct->colVal); - //} iv->type = ct->operaType; iv->colVal = tstrdup(ct->colVal); - // iv->colVal = calloc(1, strlen(ct->colVal) + 1); - // memcpy(iv->colVal, ct->colVal, strlen(ct->colVal)); taosArrayPush(iv->val, &ct->uid); } diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 16d9893778..8cb2ff9246 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -63,9 +63,9 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off } static int writeCtxGetSize(WriterCtx* ctx) { if (ctx->type == TFile) { - struct stat fstat; - stat(ctx->file.buf, &fstat); - return fstat.st_size; + int64_t file_size = 0; + taosStatFile(ctx->file.buf, &file_size, NULL); + return (int)file_size; } return 0; } @@ -99,9 +99,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); - struct stat fstat; - stat(path, &fstat); - ctx->file.size = fstat.st_size; + int64_t file_size = 0; + taosFStatFile(ctx->file.pFile, &file_size, NULL); + ctx->file.size = (int)file_size; #ifdef USE_MMAP ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); #endif @@ -142,8 +142,10 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { #endif } if (ctx->file.readOnly == false) { - struct stat fstat; - stat(ctx->file.buf, &fstat); + int64_t file_size = 0; + taosStatFile(ctx->file.buf, &file_size, NULL); + // struct stat fstat; + // stat(ctx->file.buf, &fstat); // indexError("write file size: %d", (int)(fstat.st_size)); } if (remove) { unlink(ctx->file.buf); } diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 0947c796b2..f5f46b0617 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -102,7 +102,6 @@ void tfileCacheDestroy(TFileCache* tcache) { if (tcache == NULL) { return; } - // free table cache TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); while (reader) { @@ -429,6 +428,7 @@ static bool tfileIteratorNext(Iterate* iiter) { return false; } + iv->type = ADD_VALUE; // value in tfile always ADD_VALUE iv->colVal = colVal; return true; // std::string key(ch, sz); diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c index fcaab968c2..fc28484de9 100644 --- a/source/libs/index/src/index_util.c +++ b/source/libs/index/src/index_util.c @@ -14,6 +14,7 @@ */ #include "index_util.h" #include "index.h" + typedef struct MergeIndex { int idx; int len; @@ -111,6 +112,26 @@ void iUnion(SArray *inters, SArray *final) { break; } } - tfree(mi); } + +void iExcept(SArray *total, SArray *except) { + int32_t tsz = taosArrayGetSize(total); + int32_t esz = taosArrayGetSize(except); + if (esz == 0 || tsz == 0) { + return; + } + + int vIdx = 0; + for (int i = 0; i < tsz; i++) { + uint64_t val = *(uint64_t *)taosArrayGet(total, i); + int idx = iBinarySearch(except, 0, esz - 1, val); + if (idx >= 0 && idx < esz && *(uint64_t *)taosArrayGet(except, idx) == val) { + continue; + } + taosArraySet(total, vIdx, &val); + vIdx += 1; + } + + taosArrayPopTailBatch(total, tsz - vIdx); +} diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index cb3206a611..618e20bc4b 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -19,7 +19,7 @@ static std::string fileName = "/tmp/tindex.tindex"; class FstWriter { public: FstWriter() { - remove(fileName.c_str()); + taosRemoveFile(fileName.c_str()); _wc = writerCtxCreate(TFile, fileName.c_str(), false, 64 * 1024 * 1024); _b = fstBuilderCreate(_wc, 0); } diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index e5c79d137f..df9f8b8439 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -105,6 +105,22 @@ TEST_F(JsonEnv, testWriteMillonData) { } indexMultiTermDestroy(terms); } + { + std::string colName("voltagefdadfa"); + std::string colVal("abxxxxxxxxxxxx"); + for (uint i = 0; i < 1000; i++) { + colVal[i % colVal.size()] = '0' + i % 128; + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 1000; i++) { + tIndexJsonPut(index, terms, i); + } + indexMultiTermDestroy(terms); + } + } { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc index aeff20d488..6ba8cc9525 100644 --- a/source/libs/index/test/utilUT.cc +++ b/source/libs/index/test/utilUT.cc @@ -224,3 +224,84 @@ TEST_F(UtilEnv, 04union) { iUnion(src, rslt); assert(taosArrayGetSize(rslt) == 12); } +TEST_F(UtilEnv, 01Except) { + SArray *total = taosArrayInit(4, sizeof(uint64_t)); + { + uint64_t arr1[] = {1, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(total, &arr1[i]); + } + } + + SArray *except = taosArrayInit(4, sizeof(uint64_t)); + { + uint64_t arr1[] = {1, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(except, &arr1[i]); + } + } + iExcept(total, except); + ASSERT_EQ(taosArrayGetSize(total), 0); + + taosArrayClear(total); + taosArrayClear(except); + + { + uint64_t arr1[] = {1, 4, 5, 6, 7, 8}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(total, &arr1[i]); + } + } + + { + uint64_t arr1[] = {2, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(except, &arr1[i]); + } + } + iExcept(total, except); + ASSERT_EQ(taosArrayGetSize(total), 3); + + taosArrayClear(total); + taosArrayClear(except); + { + uint64_t arr1[] = {1, 4, 5, 6, 7, 8, 10, 100}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(total, &arr1[i]); + } + } + + { + uint64_t arr1[] = {2, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(except, &arr1[i]); + } + } + iExcept(total, except); + ASSERT_EQ(taosArrayGetSize(total), 5); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 7); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 2), 8); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 3), 10); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 4), 100); + + taosArrayClear(total); + taosArrayClear(except); + { + uint64_t arr1[] = {1, 100}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(total, &arr1[i]); + } + } + + { + uint64_t arr1[] = {2, 4, 5, 6}; + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(except, &arr1[i]); + } + } + iExcept(total, except); + ASSERT_EQ(taosArrayGetSize(total), 2); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 0), 1); + ASSERT_EQ(*(uint64_t *)taosArrayGet(total, 1), 100); +} diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index 61f9980e4e..65e47ef90c 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -23,18 +23,24 @@ #include "tjson.h" typedef struct { - SRWLatch lock; - SArray *logs; // array of SMonLogItem - int32_t maxLogs; - const char *server; - uint16_t port; -} SMonitor; + int64_t ts; + ELogLevel level; + char content[MON_LOG_LEN]; +} SMonLogItem; typedef struct SMonInfo { SArray *logs; // array of SMonLogItem SJson *pJson; } SMonInfo; +typedef struct { + pthread_mutex_t lock; + SArray *logs; // array of SMonLogItem + int32_t maxLogs; + const char *server; + uint16_t port; +} SMonitor; + #ifdef __cplusplus } #endif diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monitor.c index ecf9da218b..7057f9bcd2 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monitor.c @@ -22,6 +22,19 @@ static SMonitor tsMonitor = {0}; +void monRecordLog(int64_t ts, ELogLevel level, const char *content) { + pthread_mutex_lock(&tsMonitor.lock); + int32_t size = taosArrayGetSize(tsMonitor.logs); + if (size < tsMonitor.maxLogs) { + SMonLogItem item = {.ts = ts, .level = level}; + SMonLogItem *pItem = taosArrayPush(tsMonitor.logs, &item); + if (pItem != NULL) { + tstrncpy(pItem->content, content, MON_LOG_LEN); + } + } + pthread_mutex_unlock(&tsMonitor.lock); +} + int32_t monInit(const SMonCfg *pCfg) { tsMonitor.logs = taosArrayInit(16, sizeof(SMonLogItem)); if (tsMonitor.logs == NULL) { @@ -32,24 +45,16 @@ int32_t monInit(const SMonCfg *pCfg) { tsMonitor.maxLogs = pCfg->maxLogs; tsMonitor.server = pCfg->server; tsMonitor.port = pCfg->port; - taosInitRWLatch(&tsMonitor.lock); + tsLogFp = monRecordLog; + pthread_mutex_init(&tsMonitor.lock, NULL); return 0; } void monCleanup() { + tsLogFp = NULL; taosArrayDestroy(tsMonitor.logs); tsMonitor.logs = NULL; -} - -void monAddLogItem(SMonLogItem *pItem) { - taosWLockLatch(&tsMonitor.lock); - int32_t size = taosArrayGetSize(tsMonitor.logs); - if (size >= tsMonitor.maxLogs) { - uInfo("too many logs for monitor"); - } else { - taosArrayPush(tsMonitor.logs, pItem); - } - taosWUnLockLatch(&tsMonitor.lock); + pthread_mutex_destroy(&tsMonitor.lock); } SMonInfo *monCreateMonitorInfo() { @@ -59,10 +64,10 @@ SMonInfo *monCreateMonitorInfo() { return NULL; } - taosWLockLatch(&tsMonitor.lock); + pthread_mutex_lock(&tsMonitor.lock); pMonitor->logs = taosArrayDup(tsMonitor.logs); taosArrayClear(tsMonitor.logs); - taosWUnLockLatch(&tsMonitor.lock); + pthread_mutex_unlock(&tsMonitor.lock); pMonitor->pJson = tjsonCreateObject(); if (pMonitor->pJson == NULL || pMonitor->logs == NULL) { @@ -128,11 +133,11 @@ void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { SJson *pMnodesJson = tjsonAddArrayToObject(pJson, "mnodes"); if (pMnodesJson == NULL) return; - for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { SJson *pMnodeJson = tjsonCreateObject(); if (pMnodeJson == NULL) continue; - SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->dnodes, i); + SMonMnodeDesc *pMnodeDesc = taosArrayGet(pInfo->mnodes, i); tjsonAddDoubleToObject(pMnodeJson, "mnode_id", pMnodeDesc->mnode_id); tjsonAddStringToObject(pMnodeJson, "mnode_ep", pMnodeDesc->mnode_ep); tjsonAddStringToObject(pMnodeJson, "role", pMnodeDesc->role); @@ -270,6 +275,21 @@ void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { tjsonAddDoubleToObject(pTempdirJson, "total", pInfo->tempdir.size.total); } +static const char *monLogLevelStr(ELogLevel level) { + switch (level) { + case DEBUG_ERROR: + return "error"; + case DEBUG_INFO: + return "info"; + case DEBUG_DEBUG: + return "debug"; + case DEBUG_TRACE: + return "trace"; + default: + return "undefine"; + } +} + static void monSetLogInfo(SMonInfo *pMonitor) { SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; @@ -291,7 +311,7 @@ static void monSetLogInfo(SMonInfo *pMonitor) { taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI); tjsonAddStringToObject(pLogJson, "ts", buf); - tjsonAddDoubleToObject(pLogJson, "level", pLogItem->level); + tjsonAddStringToObject(pLogJson, "level", monLogLevelStr(pLogItem->level)); tjsonAddStringToObject(pLogJson, "content", pLogItem->content); if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson); @@ -303,25 +323,25 @@ static void monSetLogInfo(SMonInfo *pMonitor) { SJson *pLogError = tjsonCreateObject(); if (pLogError == NULL) return; tjsonAddStringToObject(pLogError, "level", "error"); - tjsonAddDoubleToObject(pLogError, "total", 1); + tjsonAddDoubleToObject(pLogError, "total", tsNumOfErrorLogs); if (tjsonAddItemToArray(pSummaryJson, pLogError) != 0) tjsonDelete(pLogError); SJson *pLogInfo = tjsonCreateObject(); if (pLogInfo == NULL) return; tjsonAddStringToObject(pLogInfo, "level", "info"); - tjsonAddDoubleToObject(pLogInfo, "total", 1); + tjsonAddDoubleToObject(pLogInfo, "total", tsNumOfInfoLogs); if (tjsonAddItemToArray(pSummaryJson, pLogInfo) != 0) tjsonDelete(pLogInfo); SJson *pLogDebug = tjsonCreateObject(); if (pLogDebug == NULL) return; tjsonAddStringToObject(pLogDebug, "level", "debug"); - tjsonAddDoubleToObject(pLogDebug, "total", 1); + tjsonAddDoubleToObject(pLogDebug, "total", tsNumOfDebugLogs); if (tjsonAddItemToArray(pSummaryJson, pLogDebug) != 0) tjsonDelete(pLogDebug); SJson *pLogTrace = tjsonCreateObject(); if (pLogTrace == NULL) return; tjsonAddStringToObject(pLogTrace, "level", "trace"); - tjsonAddDoubleToObject(pLogTrace, "total", 1); + tjsonAddDoubleToObject(pLogTrace, "total", tsNumOfTraceLogs); if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace); } diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp index ad48ed5407..3eaab45e3e 100644 --- a/source/libs/monitor/test/monTest.cpp +++ b/source/libs/monitor/test/monTest.cpp @@ -193,37 +193,14 @@ void MonitorTest::GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { } void MonitorTest::AddLogInfo1() { - SMonLogItem log1 = {0}; - log1.ts = taosGetTimestampMs(); - log1.level = 1; - strcpy(log1.content, "1 -------------------------- a"); - monAddLogItem(&log1); - - SMonLogItem log2 = {0}; - log2.ts = taosGetTimestampMs(); - log2.level = 1; - strcpy(log2.content, "1 ------------------------ b"); - monAddLogItem(&log2); - - SMonLogItem log3 = {0}; - log3.ts = taosGetTimestampMs(); - log3.level = 1; - strcpy(log3.content, "1 ------- c"); - monAddLogItem(&log3); + monRecordLog(taosGetTimestampMs(), DEBUG_INFO, "1 -------------------------- a"); + monRecordLog(taosGetTimestampMs(), DEBUG_ERROR, "1 ------------------------ b"); + monRecordLog(taosGetTimestampMs(), DEBUG_DEBUG, "1 ------- c"); } void MonitorTest::AddLogInfo2() { - SMonLogItem log1; - log1.ts = taosGetTimestampMs(); - log1.level = 01; - strcpy(log1.content, "2 ------- a"); - monAddLogItem(&log1); - - SMonLogItem log2; - log2.ts = taosGetTimestampMs(); - log2.level = 0; - strcpy(log2.content, "2 ------- b"); - monAddLogItem(&log2); + monRecordLog(taosGetTimestampMs(), DEBUG_ERROR, "2 ------- a"); + monRecordLog(taosGetTimestampMs(), DEBUG_ERROR, "2 ------- b"); } TEST_F(MonitorTest, 01_Full) { diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 745982e869..4b58bc4e69 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -153,7 +153,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); + taosHashGetDup(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; dst->size = src->size; TSWAP(dst->pData, src->pData, char*); diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 457d586ea4..e626fc68ae 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -13,19 +13,17 @@ * along with this program. If not, see . */ -#include "mockCatalog.h" - #include - #include "stub.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat" -#include "addr_any.h" +#include #pragma GCC diagnostic pop +#include "mockCatalog.h" namespace { void generateTestT1(MockCatalogService* mcs) { diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 00d64bd12a..cdb547ee1b 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -15,10 +15,10 @@ #include "mockCatalogService.h" -#include "tep.h" #include #include #include +#include "tdatablock.h" #include "tname.h" #include "ttypes.h" diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index b971760132..b80efe09ed 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -#include #include #include -#include "tglobal.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -25,6 +23,8 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "function.h" +#include "tglobal.h" #include "astGenerator.h" #include "parserInt.h" #include "taos.h" diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 43daff9fec..095e2f4f97 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include #include #include @@ -25,6 +24,7 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" +#include "function.h" #include "astGenerator.h" #include "parserInt.h" #include "taos.h" diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index a2165453d5..63fbf59c06 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -1,23 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include "os.h" -#include "tmsg.h" #include "query.h" #include "tglobal.h" -#include "tsched.h" +#include "tmsg.h" #include "trpc.h" +#include "tsched.h" -#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) -#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) +#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) +#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) static struct SSchema _s = { .colId = TSDB_TBNAME_COLUMN_INDEX, - .type = TSDB_DATA_TYPE_BINARY, + .type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .name = "tbname", }; -const SSchema* tGetTbnameColumnSchema() { - return &_s; -} +const SSchema* tGetTbnameColumnSchema() { return &_s; } static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { int32_t rowLen = 0; @@ -87,7 +100,7 @@ int32_t initTaskQueue() { double factor = 4.0; int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); - + int32_t queueSize = tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); if (NULL == pTaskQueue) { @@ -96,19 +109,21 @@ int32_t initTaskQueue() { } qDebug("task queue is initialized, numOfThreads: %d", numOfThreads); + return 0; } int32_t cleanupTaskQueue() { taosCleanUpScheduler(pTaskQueue); + return 0; } static void execHelper(struct SSchedMsg* pSchedMsg) { assert(pSchedMsg != NULL && pSchedMsg->ahandle != NULL); - __async_exec_fn_t execFn = (__async_exec_fn_t) pSchedMsg->ahandle; - int32_t code = execFn(pSchedMsg->thandle); + __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; + int32_t code = execFn(pSchedMsg->thandle); if (code != 0 && pSchedMsg->msg != NULL) { - *(int32_t*) pSchedMsg->msg = code; + *(int32_t*)pSchedMsg->msg = code; } } @@ -116,34 +131,33 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) assert(execFn != NULL); SSchedMsg schedMsg = {0}; - schedMsg.fp = execHelper; + schedMsg.fp = execHelper; schedMsg.ahandle = execFn; schedMsg.thandle = execParam; - schedMsg.msg = code; + schedMsg.msg = code; taosScheduleTask(pTaskQueue, &schedMsg); + return 0; } -int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { - char *pMsg = rpcMallocCont(pInfo->msgInfo.len); +int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) { + char* pMsg = rpcMallocCont(pInfo->msgInfo.len); if (NULL == pMsg) { - qError("0x%"PRIx64" msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); + qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return terrno; } memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len); - SRpcMsg rpcMsg = { - .msgType = pInfo->msgType, - .pCont = pMsg, - .contLen = pInfo->msgInfo.len, - .ahandle = (void*) pInfo, - .handle = pInfo->msgInfo.handle, - .code = 0 - }; + SRpcMsg rpcMsg = {.msgType = pInfo->msgType, + .pCont = pMsg, + .contLen = pInfo->msgInfo.len, + .ahandle = (void*)pInfo, + .handle = pInfo->msgInfo.handle, + .code = 0}; assert(pInfo->fp != NULL); rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 8ad5a76388..231f0c7fff 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -14,7 +14,6 @@ */ #include -#include #include #pragma GCC diagnostic push @@ -26,18 +25,19 @@ #pragma GCC diagnostic ignored "-Wformat" #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-arith" +#include #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" -#include "tep.h" +#include "tdatablock.h" #include "trpc.h" #include "planner.h" #include "qworker.h" #include "stub.h" -#include "addr_any.h" #include "executor.h" #include "dataSinkMgt.h" diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index b4fe9a67ca..e1ffb2efd1 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -20,13 +20,13 @@ extern "C" { #endif +#include "query.h" +#include "querynodes.h" +#include "scalar.h" +#include "tcommon.h" +#include "tdatablock.h" #include "thash.h" #include "tname.h" -#include "tcommon.h" -#include "scalar.h" -#include "querynodes.h" -#include "query.h" -#include "tep.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e5be22c0a8..a7aea5a7e5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -16,11 +16,11 @@ #include #include "thash.h" //#include "queryLog.h" -#include "tcompare.h" +#include "filter.h" #include "filterInt.h" #include "sclInt.h" -#include "filter.h" -#include "tep.h" +#include "tcompare.h" +#include "tdatablock.h" OptrStr gOptrStr[] = { {0, "invalid"}, diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 0245226dfc..b8cdda9ed1 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1,11 +1,11 @@ -#include "nodes.h" -#include "tcommon.h" -#include "querynodes.h" #include "function.h" #include "functionMgt.h" -#include "sclvector.h" +#include "nodes.h" +#include "querynodes.h" #include "sclInt.h" -#include "tep.h" +#include "sclvector.h" +#include "tcommon.h" +#include "tdatablock.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 3c431ff33f..b066dd2e77 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -15,15 +15,15 @@ #include "os.h" -#include "ttypes.h" -#include "sclvector.h" -#include "tcompare.h" -#include "querynodes.h" +#include "filter.h" #include "filterInt.h" #include "query.h" +#include "querynodes.h" #include "sclInt.h" -#include "tep.h" -#include "filter.h" +#include "sclvector.h" +#include "tcompare.h" +#include "tdatablock.h" +#include "ttypes.h" //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index fafc1ea42e..08210aa2f0 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -14,7 +14,6 @@ */ #include -#include #include #pragma GCC diagnostic push @@ -26,19 +25,26 @@ #pragma GCC diagnostic ignored "-Wformat" #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-arith" +#include #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" -#include "tep.h" +#include "tdatablock.h" #include "stub.h" -#include "addr_any.h" #include "scalar.h" -#include "nodes.h" -#include "tlog.h" #include "filter.h" +#include "nodes.h" +#include "scalar.h" +#include "stub.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" +#include "tlog.h" +#include "tvariant.h" namespace { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index b9aef99088..faf13f0a82 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -14,7 +14,6 @@ */ #include -#include #include #pragma GCC diagnostic push @@ -26,15 +25,16 @@ #pragma GCC diagnostic ignored "-Wformat" #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-arith" +#include #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" -#include "tep.h" +#include "tdatablock.h" #include "stub.h" -#include "addr_any.h" #include "scalar.h" #include "nodes.h" #include "tlog.h" diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 11ed3335e6..daf65cf251 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -14,18 +14,31 @@ */ #include -#include #include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wreturn-type" +#pragma GCC diagnostic ignored "-Wformat" +#include + + #include "os.h" +#include "tglobal.h" #include "taos.h" #include "tdef.h" #include "tvariant.h" #include "catalog.h" #include "scheduler.h" -#include "tep.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" #include "trpc.h" +#include "tvariant.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -37,7 +50,6 @@ #include "schedulerInt.h" #include "stub.h" -#include "addr_any.h" #include "tref.h" namespace { diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 2b62f9b8b3..15ed5503eb 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -29,43 +29,41 @@ extern "C" { #include "tlog.h" #include "ttimer.h" -extern int32_t sDebugFlag; - -#define sFatal(...) \ - { \ - if (sDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sFatal(...) \ + { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("SYN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ } -#define sError(...) \ - { \ - if (sDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sError(...) \ + { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("SYN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ } -#define sWarn(...) \ - { \ - if (sDebugFlag & DEBUG_WARN) { \ - taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sWarn(...) \ + { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("SYN WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ } -#define sInfo(...) \ - { \ - if (sDebugFlag & DEBUG_INFO) { \ - taosPrintLog("SYN INFO ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sInfo(...) \ + { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("SYN INFO ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ } -#define sDebug(...) \ - { \ - if (sDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("SYN DEBUG ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sDebug(...) \ + { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("SYN DEBUG ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \ + } \ } -#define sTrace(...) \ - { \ - if (sDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("SYN TRACE ", sDebugFlag, __VA_ARGS__); \ - } \ +#define sTrace(...) \ + { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("SYN TRACE ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \ + } \ } struct SRaft; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 55c4d57201..25775bec76 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -14,7 +14,7 @@ */ #include "syncIO.h" -#include +#include #include "syncOnMessage.h" #include "tglobal.h" #include "ttimer.h" diff --git a/source/libs/sync/test/syncIOSendMsgClientTest.cpp b/source/libs/sync/test/syncIOSendMsgClientTest.cpp index 9e8c7c8b65..0d06c4f811 100644 --- a/source/libs/sync/test/syncIOSendMsgClientTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgClientTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOSendMsgServerTest.cpp b/source/libs/sync/test/syncIOSendMsgServerTest.cpp index 8af9344ed6..1582e097d3 100644 --- a/source/libs/sync/test/syncIOSendMsgServerTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgServerTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index a297981ee5..9b2cfcf1d8 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 1559b57585..42b9a73432 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 90304079e3..3d31c596bf 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index e533b89a92..9cfbe5a4ea 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -1,6 +1,6 @@ -#include "syncRaftStore.h" #include -#include "gtest/gtest.h" +#include +#include "syncRaftStore.h" #include "syncIO.h" #include "syncInt.h" diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 0b397ef921..c1c5658aba 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,5 +1,5 @@ #include -#include "gtest/gtest.h" +#include #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" diff --git a/source/libs/tdb/src/db/tdb.c b/source/libs/tdb/src/db/tdb.c index cc3b7fa6b9..65d4cf80cc 100644 --- a/source/libs/tdb/src/db/tdb.c +++ b/source/libs/tdb/src/db/tdb.c @@ -90,7 +90,7 @@ int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) { // get page file from the env, if not opened yet, open it pPgFile = NULL; snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname); - fileExist = (tdbCheckFileAccess(fname, TDB_F_OK) == 0); + fileExist = taosCheckExistFile(fname); if (fileExist) { tdbGnrtFileID(dbfname, fileid, false); pPgFile = tdbEnvGetPageFile(pEnv, fileid); diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index fa9a3297da..fe0f3befd6 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -16,16 +16,16 @@ #include "tdbInt.h" int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { - struct stat statbuf; + int64_t stDev = 0, stIno = 0; - if (stat(fname, &statbuf) < 0) { + if (taosDevInoFile(fname, &stDev, &stIno) < 0) { return -1; } memset(fileid, 0, TDB_FILE_ID_LEN); - ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino; - ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev; + ((uint64_t *)fileid)[0] = stDev; + ((uint64_t *)fileid)[1] = stIno; if (unique) { ((uint64_t *)fileid)[2] = rand(); } @@ -33,35 +33,34 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) { return 0; } -int tdbCheckFileAccess(const char *pathname, int mode) { - int flags = 0; +// int tdbCheckFileAccess(const char *pathname, int mode) { +// int flags = 0; - if (mode & TDB_F_OK) { - flags |= F_OK; - } +// if (mode & TDB_F_OK) { +// flags |= F_OK; +// } - if (mode & TDB_R_OK) { - flags |= R_OK; - } +// if (mode & TDB_R_OK) { +// flags |= R_OK; +// } - if (mode & TDB_W_OK) { - flags |= W_OK; - } +// if (mode & TDB_W_OK) { +// flags |= W_OK; +// } - return access(pathname, flags); -} +// return access(pathname, flags); +// } int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) { - struct stat st; int ret; - - ret = stat(fname, &st); + int64_t file_size = 0; + ret = taosStatFile(fname, &file_size, NULL); if (ret != 0) { return -1; } - ASSERT(st.st_size % pgSize == 0); + ASSERT(file_size % pgSize == 0); - *pSize = st.st_size / pgSize; + *pSize = file_size / pgSize; return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 8108e5aba6..ca05790f77 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -30,10 +30,10 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); -#define TDB_F_OK 0x1 -#define TDB_R_OK 0x2 -#define TDB_W_OK 0x4 -int tdbCheckFileAccess(const char *pathname, int mode); +// #define TDB_F_OK 0x1 +// #define TDB_R_OK 0x2 +// #define TDB_W_OK 0x4 +// int tdbCheckFileAccess(const char *pathname, int mode); int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize); diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 5ab0b4c0f1..ad550c7804 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,4 +1,4 @@ -#include "gtest/gtest.h" +#include #include "tdb.h" diff --git a/source/libs/tfs/CMakeLists.txt b/source/libs/tfs/CMakeLists.txt index 607ccd4c48..97e02fc8a9 100644 --- a/source/libs/tfs/CMakeLists.txt +++ b/source/libs/tfs/CMakeLists.txt @@ -6,7 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(tfs os util common) +target_link_libraries(tfs os util common monitor) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/libs/tfs/inc/tfsInt.h b/source/libs/tfs/inc/tfsInt.h index cfc246f07b..913f34d6c2 100644 --- a/source/libs/tfs/inc/tfsInt.h +++ b/source/libs/tfs/inc/tfsInt.h @@ -25,15 +25,13 @@ #include "thash.h" #include "tlog.h" -extern int32_t fsDebugFlag; - // For debug purpose -#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }} -#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }} -#define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }} -#define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", 255, __VA_ARGS__); }} -#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }} -#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", fsDebugFlag, __VA_ARGS__); }} +#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", DEBUG_DEBUG, fsDebugFlag, __VA_ARGS__); }} +#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", DEBUG_TRACE, fsDebugFlag, __VA_ARGS__); }} typedef struct { int32_t level; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index f686703643..2579490791 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -202,7 +202,7 @@ void tfsDirname(const STfsFile *pFile, char *dest) { tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); } -int32_t tfsRemoveFile(const STfsFile *pFile) { return remove(pFile->aname); } +int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); } int32_t tfsCopyFile(const STfsFile *pFile1, const STfsFile *pFile2) { return taosCopyFile(pFile1->aname, pFile2->aname); @@ -389,7 +389,6 @@ static int32_t tfsMount(STfs *pTfs, SDiskCfg *pCfg) { static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { char dirName[TSDB_FILENAME_LEN] = "\0"; - struct stat pstat; if (pCfg->level < 0 || pCfg->level >= TFS_MAX_TIERS) { fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); @@ -422,19 +421,13 @@ static int32_t tfsCheckAndFormatCfg(STfs *pTfs, SDiskCfg *pCfg) { return -1; } - if (access(dirName, W_OK | R_OK | F_OK) != 0) { + if (!taosCheckAccessFile(dirName, TD_FILE_ACCESS_EXIST_OK | TD_FILE_ACCESS_READ_OK | TD_FILE_ACCESS_WRITE_OK)) { fError("failed to mount %s to FS since no R/W access rights", pCfg->dir); terrno = TSDB_CODE_FS_INVLD_CFG; return -1; } - if (stat(dirName, &pstat) < 0) { - fError("failed to mount %s to FS since %s", pCfg->dir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (!S_ISDIR(pstat.st_mode)) { + if (!taosIsDir(dirName)) { fError("failed to mount %s to FS since not a directory", pCfg->dir); terrno = TSDB_CODE_FS_INVLD_CFG; return -1; @@ -544,3 +537,24 @@ static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter) { return pDisk; } + +int32_t tfsGetMonitorInfo(STfs *pTfs, SMonDiskInfo *pInfo) { + pInfo->datadirs = taosArrayInit(32, sizeof(SMonDiskDesc)); + if (pInfo->datadirs == NULL) return -1; + + tfsUpdateSize(pTfs); + + tfsLock(pTfs); + for (int32_t level = 0; level < pTfs->nlevel; level++) { + STfsTier *pTier = &pTfs->tiers[level]; + for (int32_t disk = 0; disk < pTier->ndisk; ++disk) { + STfsDisk *pDisk = pTier->disks[disk]; + SMonDiskDesc dinfo = {0}; + dinfo.size = pDisk->size; + dinfo.level = pDisk->level; + tstrncpy(dinfo.name, pDisk->path, sizeof(dinfo.name)); + taosArrayPush(pInfo->datadirs, &dinfo); + } + } + tfsUnLock(pTfs); +} \ No newline at end of file diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index af66304f84..9cb914a670 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -119,29 +119,29 @@ TEST_F(TfsTest, 03_Dir) { char p1[] = "p1"; char ap1[128] = {0}; snprintf(ap1, 128, "%s%s%s", root, TD_DIRSEP, p1); - EXPECT_NE(taosDirExist(ap1), 0); + EXPECT_NE(taosDirExist(ap1), 1); EXPECT_EQ(tfsMkdir(pTfs, p1), 0); - EXPECT_EQ(taosDirExist(ap1), 0); + EXPECT_EQ(taosDirExist(ap1), 1); char p2[] = "p2"; char ap2[128] = {0}; snprintf(ap2, 128, "%s%s%s", root, TD_DIRSEP, p2); SDiskID did = {0}; - EXPECT_NE(taosDirExist(ap2), 0); + EXPECT_NE(taosDirExist(ap2), 1); EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); - EXPECT_EQ(taosDirExist(ap2), 0); + EXPECT_EQ(taosDirExist(ap2), 1); char p3[] = "p3/p2/p1/p0"; char ap3[128] = {0}; snprintf(ap3, 128, "%s%s%s", root, TD_DIRSEP, p3); - EXPECT_NE(taosDirExist(ap3), 0); + EXPECT_NE(taosDirExist(ap3), 1); EXPECT_NE(tfsMkdir(pTfs, p3), 0); EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); - EXPECT_EQ(taosDirExist(ap3), 0); + EXPECT_EQ(taosDirExist(ap3), 1); EXPECT_EQ(tfsRmdir(pTfs, p3), 0); - EXPECT_NE(taosDirExist(ap3), 0); + EXPECT_NE(taosDirExist(ap3), 1); char p45[] = "p5"; char p44[] = "p4"; @@ -149,12 +149,12 @@ TEST_F(TfsTest, 03_Dir) { char ap4[128] = {0}; snprintf(ap4, 128, "%s%s%s", root, TD_DIRSEP, p4); - EXPECT_NE(taosDirExist(ap4), 0); + EXPECT_NE(taosDirExist(ap4), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); - EXPECT_EQ(taosDirExist(ap4), 0); + EXPECT_EQ(taosDirExist(ap4), 1); EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); - EXPECT_NE(taosDirExist(ap4), 0); + EXPECT_NE(taosDirExist(ap4), 1); tfsClose(pTfs); } @@ -251,9 +251,9 @@ TEST_F(TfsTest, 04_File) { char af2[128] = {0}; snprintf(af2, 128, "%s%s%s", root, TD_DIRSEP, n2); - EXPECT_EQ(taosDirExist(af2), 0); + EXPECT_EQ(taosDirExist(af2), 1); tfsRemoveFile(&f2); - EXPECT_NE(taosDirExist(af2), 0); + EXPECT_NE(taosDirExist(af2), 1); { STfsDir *pDir = tfsOpendir(pTfs, "t3"); @@ -529,35 +529,35 @@ TEST_F(TfsTest, 05_MultiDisk) { snprintf(ap22, 128, "%s%s%s", root22, TD_DIRSEP, p1); char ap23[128] = {0}; snprintf(ap23, 128, "%s%s%s", root23, TD_DIRSEP, p1); - EXPECT_NE(taosDirExist(ap00), 0); - EXPECT_NE(taosDirExist(ap01), 0); - EXPECT_NE(taosDirExist(ap10), 0); - EXPECT_NE(taosDirExist(ap11), 0); - EXPECT_NE(taosDirExist(ap12), 0); - EXPECT_NE(taosDirExist(ap20), 0); - EXPECT_NE(taosDirExist(ap21), 0); - EXPECT_NE(taosDirExist(ap22), 0); - EXPECT_NE(taosDirExist(ap23), 0); + EXPECT_NE(taosDirExist(ap00), 1); + EXPECT_NE(taosDirExist(ap01), 1); + EXPECT_NE(taosDirExist(ap10), 1); + EXPECT_NE(taosDirExist(ap11), 1); + EXPECT_NE(taosDirExist(ap12), 1); + EXPECT_NE(taosDirExist(ap20), 1); + EXPECT_NE(taosDirExist(ap21), 1); + EXPECT_NE(taosDirExist(ap22), 1); + EXPECT_NE(taosDirExist(ap23), 1); EXPECT_EQ(tfsMkdir(pTfs, p1), 0); - EXPECT_EQ(taosDirExist(ap00), 0); - EXPECT_EQ(taosDirExist(ap01), 0); - EXPECT_EQ(taosDirExist(ap10), 0); - EXPECT_EQ(taosDirExist(ap11), 0); - EXPECT_EQ(taosDirExist(ap12), 0); - EXPECT_EQ(taosDirExist(ap20), 0); - EXPECT_EQ(taosDirExist(ap21), 0); - EXPECT_EQ(taosDirExist(ap22), 0); - EXPECT_EQ(taosDirExist(ap23), 0); + EXPECT_EQ(taosDirExist(ap00), 1); + EXPECT_EQ(taosDirExist(ap01), 1); + EXPECT_EQ(taosDirExist(ap10), 1); + EXPECT_EQ(taosDirExist(ap11), 1); + EXPECT_EQ(taosDirExist(ap12), 1); + EXPECT_EQ(taosDirExist(ap20), 1); + EXPECT_EQ(taosDirExist(ap21), 1); + EXPECT_EQ(taosDirExist(ap22), 1); + EXPECT_EQ(taosDirExist(ap23), 1); EXPECT_EQ(tfsRmdir(pTfs, p1), 0); - EXPECT_NE(taosDirExist(ap00), 0); - EXPECT_NE(taosDirExist(ap01), 0); - EXPECT_NE(taosDirExist(ap10), 0); - EXPECT_NE(taosDirExist(ap11), 0); - EXPECT_NE(taosDirExist(ap12), 0); - EXPECT_NE(taosDirExist(ap20), 0); - EXPECT_NE(taosDirExist(ap21), 0); - EXPECT_NE(taosDirExist(ap22), 0); - EXPECT_NE(taosDirExist(ap23), 0); + EXPECT_NE(taosDirExist(ap00), 1); + EXPECT_NE(taosDirExist(ap01), 1); + EXPECT_NE(taosDirExist(ap10), 1); + EXPECT_NE(taosDirExist(ap11), 1); + EXPECT_NE(taosDirExist(ap12), 1); + EXPECT_NE(taosDirExist(ap20), 1); + EXPECT_NE(taosDirExist(ap21), 1); + EXPECT_NE(taosDirExist(ap22), 1); + EXPECT_NE(taosDirExist(ap23), 1); char p2[] = "p2"; char _ap21[128] = {0}; @@ -565,22 +565,22 @@ TEST_F(TfsTest, 05_MultiDisk) { SDiskID did = {0}; did.level = 2; did.id = 1; - EXPECT_NE(taosDirExist(_ap21), 0); + EXPECT_NE(taosDirExist(_ap21), 1); EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); - EXPECT_EQ(taosDirExist(_ap21), 0); + EXPECT_EQ(taosDirExist(_ap21), 1); char p3[] = "p3/p2/p1/p0"; char _ap12[128] = {0}; snprintf(_ap12, 128, "%s%s%s", root12, TD_DIRSEP, p3); did.level = 1; did.id = 2; - EXPECT_NE(taosDirExist(_ap12), 0); + EXPECT_NE(taosDirExist(_ap12), 1); EXPECT_NE(tfsMkdir(pTfs, p3), 0); EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); - EXPECT_EQ(taosDirExist(_ap12), 0); + EXPECT_EQ(taosDirExist(_ap12), 1); EXPECT_EQ(tfsRmdir(pTfs, p3), 0); - EXPECT_NE(taosDirExist(_ap12), 0); + EXPECT_NE(taosDirExist(_ap12), 1); char p45[] = "p5"; char p44[] = "p4"; @@ -590,12 +590,12 @@ TEST_F(TfsTest, 05_MultiDisk) { did.level = 2; did.id = 2; - EXPECT_NE(taosDirExist(_ap22), 0); + EXPECT_NE(taosDirExist(_ap22), 1); EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); - EXPECT_EQ(taosDirExist(_ap22), 0); + EXPECT_EQ(taosDirExist(_ap22), 1); EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); EXPECT_EQ(tfsRmdir(pTfs, p4), 0); - EXPECT_NE(taosDirExist(_ap22), 0); + EXPECT_NE(taosDirExist(_ap22), 1); } //------------- File -----------------// @@ -660,7 +660,7 @@ TEST_F(TfsTest, 05_MultiDisk) { char af2[128] = {0}; snprintf(af2, 128, "%s%s%s", root23, TD_DIRSEP, n2); - EXPECT_EQ(taosDirExist(af2), 0); + EXPECT_EQ(taosDirExist(af2), 1); tfsRemoveFile(&f2); { @@ -678,7 +678,7 @@ TEST_F(TfsTest, 05_MultiDisk) { tfsClosedir(pDir); } - EXPECT_NE(taosDirExist(af2), 0); + EXPECT_NE(taosDirExist(af2), 1); EXPECT_GT(tfsCopyFile(&f1, &f2), 0); { diff --git a/source/libs/transport/inc/rpcLog.h b/source/libs/transport/inc/rpcLog.h index 621504091c..a5db866932 100644 --- a/source/libs/transport/inc/rpcLog.h +++ b/source/libs/transport/inc/rpcLog.h @@ -22,44 +22,41 @@ extern "C" { #include "tlog.h" -extern int32_t rpcDebugFlag; - -// rpcDebugFlag = 143 -#define tFatal(...) \ - { \ - if (rpcDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("RPC FATAL ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tFatal(...) \ + { \ + if (rpcDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("RPC FATAL ", DEBUG_FATAL, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tError(...) \ - { \ - if (rpcDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("RPC ERROR ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tError(...) \ + { \ + if (rpcDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("RPC ERROR ", DEBUG_ERROR, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tWarn(...) \ - { \ - if (rpcDebugFlag & DEBUG_WARN) { \ - taosPrintLog("RPC WARN ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tWarn(...) \ + { \ + if (rpcDebugFlag & DEBUG_WARN) { \ + taosPrintLog("RPC WARN ", DEBUG_WARN, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tInfo(...) \ - { \ - if (rpcDebugFlag & DEBUG_INFO) { \ - taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tInfo(...) \ + { \ + if (rpcDebugFlag & DEBUG_INFO) { \ + taosPrintLog("RPC ", DEBUG_INFO, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tDebug(...) \ - { \ - if (rpcDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tDebug(...) \ + { \ + if (rpcDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("RPC ", DEBUG_DEBUG, rpcDebugFlag, __VA_ARGS__); \ + } \ } -#define tTrace(...) \ - { \ - if (rpcDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); \ - } \ +#define tTrace(...) \ + { \ + if (rpcDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); \ + } \ } #define tDump(x, y) \ { \ diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index 4842a0c800..f4babc9980 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index a1c181ac98..c763b7bd8a 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) { if (pDataFile != NULL) { taosCloseFile(&pDataFile); - remove(dataName); + taosRemoveFile(dataName); } return 0; diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index bcdf32bf6a..6fc935cb61 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 5432a07649..da5284b5c5 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -181,7 +181,7 @@ int main(int argc, char *argv[]) { if (pDataFile != NULL) { taosCloseFile(&pDataFile); - remove(dataName); + taosRemoveFile(dataName); } return 0; diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index b7ef296b9d..f43fa7aae6 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index 6db709da51..5edddb006b 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -15,10 +15,10 @@ #include #include #include -#include "tep.h" +#include "tdatablock.h" #include "tglobal.h" -#include "trpc.h" #include "tlog.h" +#include "trpc.h" using namespace std; const char *label = "APP"; @@ -148,7 +148,7 @@ class TransObj { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - tscEmbeddedInUtil = 1; + tsLogEmbedded = 1; tsAsyncLog = 0; std::string path = "/tmp/transport"; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index ae0b0bd849..2a4f3497e4 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -64,10 +64,10 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); - struct stat statbuf; - stat(fnameStr, &statbuf); - int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); - pLastFileInfo->fileSize = statbuf.st_size; + int64_t file_size = 0; + taosStatFile(fnameStr, &file_size, NULL); + int readSize = TMIN(WAL_MAX_SIZE + 2, file_size); + pLastFileInfo->fileSize = file_size; TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pFile == NULL) { @@ -177,11 +177,11 @@ int walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz-1); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); - struct stat statbuf; - stat(fnameStr, &statbuf); + int64_t file_size = 0; + taosStatFile(fnameStr, &file_size, NULL); - if (oldSz != newSz || pLastFileInfo->fileSize != statbuf.st_size) { - pLastFileInfo->fileSize = statbuf.st_size; + if (oldSz != newSz || pLastFileInfo->fileSize != file_size) { + pLastFileInfo->fileSize = file_size; pWal->vers.lastVer = walScanLogGetLastVer(pWal); ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer; ASSERT(pWal->vers.lastVer != -1); @@ -379,7 +379,7 @@ int walSaveMeta(SWal* pWal) { // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); } free(serialized); return 0; @@ -395,9 +395,9 @@ int walLoadMeta(SWal* pWal) { char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); // read metafile - struct stat statbuf; - stat(fnameStr, &statbuf); - int size = statbuf.st_size; + int64_t file_size = 0; + taosStatFile(fnameStr, &file_size, NULL); + int size = (int)file_size; char* buf = malloc(size + 5); if (buf == NULL) { terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 4b1f0ba306..699da75790 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -55,9 +55,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); for (int i = pWal->writeCur; i < fileSetSize; i++) { walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); walBuildIdxName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); } // pop from fileInfoSet taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1); @@ -162,8 +162,8 @@ int32_t walEndSnapshot(SWal *pWal) { } // iterate files, until the searched result for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if ((pWal->cfg.retentionSize != -1 && pWal->totSize > pWal->cfg.retentionSize) - || (pWal->cfg.retentionPeriod != -1 && iter->closeTs + pWal->cfg.retentionPeriod > ts)) { + if ((pWal->cfg.retentionSize != -1 && pWal->totSize > pWal->cfg.retentionSize) || + (pWal->cfg.retentionPeriod != -1 && iter->closeTs + pWal->cfg.retentionPeriod > ts)) { // delete according to file size or close time deleteCnt++; newTotSize -= iter->fileSize; @@ -174,9 +174,9 @@ int32_t walEndSnapshot(SWal *pWal) { for (int i = 0; i < deleteCnt; i++) { SWalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i); walBuildLogName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); walBuildIdxName(pWal, pInfo->firstVer, fnameStr); - remove(fnameStr); + taosRemoveFile(fnameStr); } // make new array, remove files @@ -279,6 +279,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in } else { // reject skip log or rewrite log // must truncate explicitly first + terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } /*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/ @@ -303,16 +304,18 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { // ftruncate - code = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + return -1; } if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { // ftruncate - code = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + return -1; } code = walWriteIndex(pWal, index, offset); @@ -329,7 +332,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in pthread_mutex_unlock(&pWal->mutex); - return code; + return 0; } void walFsync(SWal *pWal, bool forceFsync) { diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index b65a200ca1..230555e016 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -339,9 +339,9 @@ TEST_F(WalRetentionEnv, repairMeta1) { //getchar(); char buf[100]; sprintf(buf, "%s/meta-ver%d", pathName, 0); - remove(buf); + taosRemoveFile(buf); sprintf(buf, "%s/meta-ver%d", pathName, 1); - remove(buf); + taosRemoveFile(buf); SetUp(); //getchar(); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 91ef97e66b..7d7382d83f 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -49,7 +49,7 @@ void taosRemoveDir(const char *dirname) { if (de->d_type & DT_DIR) { taosRemoveDir(filename); } else { - (void)remove(filename); + (void)taosRemoveFile(filename); //printf("file:%s is removed\n", filename); } } @@ -60,7 +60,7 @@ void taosRemoveDir(const char *dirname) { //printf("dir:%s is removed\n", dirname); } -int32_t taosDirExist(char *dirname) { return access(dirname, F_OK); } +bool taosDirExist(char *dirname) { return taosCheckExistFile(dirname); } int32_t taosMkDir(const char *dirname) { int32_t code = mkdir(dirname, 0755); @@ -102,7 +102,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { if (fileSec <= 100) continue; int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1); if (days > keepDays) { - (void)remove(filename); + (void)taosRemoveFile(filename); //printf("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); } else { //printf("file:%s won't be removed, days:%d keepDays:%d", filename, days, keepDays); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 4c368fe895..0b2fe904b3 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -31,11 +31,11 @@ char tsLocale[TD_LOCALE_LEN] = {0}; char tsCharset[TD_CHARSET_LEN] = {0}; int8_t tsDaylight = 0; bool tsEnableCoreFile = 0; -int64_t tsPageSize = 0; +int64_t tsPageSizeKB = 0; int64_t tsOpenMax = 0; int64_t tsStreamMax = 0; -int32_t tsNumOfCores = 0; -int32_t tsTotalMemoryMB = 0; +float tsNumOfCores = 0; +int64_t tsTotalMemoryKB = 0; void osInit() { srand(taosSafeRand()); @@ -44,6 +44,11 @@ void osInit() { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); taosGetSystemInfo(); + // deadlock in query + if (tsNumOfCores < 2) { + tsNumOfCores = 2; + } + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) taosWinSocketInit(); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c2e9b52f88..652e0b5182 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -142,11 +142,13 @@ int64_t taosCopyFile(const char *from, const char *to) { _err: if (pFileFrom != NULL) taosCloseFile(&pFileFrom); if (pFileTo != NULL) taosCloseFile(&pFileTo); - remove(to); + taosRemoveFile(to); return -1; #endif } +int32_t taosRemoveFile(const char *path) { return remove(path); } + int32_t taosRenameFile(const char *oldName, const char *newName) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); @@ -186,6 +188,27 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { return 0; #endif } +int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); + if (code < 0) { + return code; + } + + if (stDev != NULL) { + *stDev = fileStat.st_dev; + } + + if (stIno != NULL) { + *stIno = fileStat.st_ino; + } + + return 0; +#endif +} void autoDelFileListAdd(const char *path) { return; } @@ -733,3 +756,21 @@ int32_t taosEOFFile(TdFilePtr pFile) { return feof(pFile->fp); } +bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { + int flags = 0; + + if (tdFileAccessOptions & TD_FILE_ACCESS_EXIST_OK) { + flags |= F_OK; + } + + if (tdFileAccessOptions & TD_FILE_ACCESS_READ_OK) { + flags |= R_OK; + } + + if (tdFileAccessOptions & TD_FILE_ACCESS_WRITE_OK) { + flags |= W_OK; + } + + return access(pathname, flags) == 0; +} +bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); }; \ No newline at end of file diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 3e761c6d91..8f2820f67a 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -39,32 +39,32 @@ #include #pragma warning(pop) -static int32_t taosGetTotalMemory() { +int32_t taosGetTotalMemory(int64_t *totalKB) { MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { - return 0; + return -1; } - float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); - return (int32_t)nMemTotal; + *totalKB = memsStat.ullTotalPhys / 1024; + return 0; } -bool taosGetSysMemory(float *memoryUsedMB) { +int32_t taosGetSysMemory(int64_t *usedKB) { MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { - return false; + return -1; } - float nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); - float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + int64_t nMemFree = memsStat.ullAvailPhys / 1024; + int64_t nMemTotal = memsStat.ullTotalPhys / 1024.0; - *memoryUsedMB = nMemTotal - nMemFree; - return true; + *usedKB = nMemTotal - nMemFree; + return 0; } -bool taosGetProcMemory(float *memoryUsedMB) { +int32_t taosGetProcMemory(int64_t *usedKB) { unsigned bytes_used = 0; #if defined(_WIN64) && defined(_MSC_VER) @@ -76,21 +76,21 @@ bool taosGetProcMemory(float *memoryUsedMB) { } #endif - *memoryUsedMB = (float)bytes_used / 1024 / 1024; - return true; + *usedKB = bytes_used / 1024; + return 0; } - -int32_t taosGetCpuCores() { +int32_t taosGetCpuCores(float *numOfCores) { SYSTEM_INFO info; GetSystemInfo(&info); - return (int32_t)info.dwNumberOfProcessors; + *numOfCores = info.dwNumberOfProcessors; + return 0; } -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { +int32_t taosGetCpuUsage(double *sysCpuUsage, double *procCpuUsage) { *sysCpuUsage = 0; *procCpuUsage = 0; - return true; + return 0; } int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { @@ -106,79 +106,47 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { diskSize->used = (int64_t)(i64TotalBytes - i64FreeBytes); return 0; } else { - //printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { - if (bytes) *bytes = 0; - if (rbytes) *rbytes = 0; - if (tbytes) *tbytes = 0; - return true; +int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { + *receive_bytes = 0; + *transmit_bytes = 0; + return 0; } -bool taosGetBandSpeed(float *bandSpeedKb) { - *bandSpeedKb = 0; - return true; -} - -bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { +int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { IO_COUNTERS io_counter; if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) { - if (readbyte) *readbyte = io_counter.ReadTransferCount; - if (writebyte) *writebyte = io_counter.WriteTransferCount; - return true; + if (rchars) *rchars = io_counter.ReadTransferCount; + if (wchars) *wchars = io_counter.WriteTransferCount; + if (read_bytes) *read_bytes = 0; + if (write_bytes) *write_bytes = 0; + return 0; } - return false; -} - -bool taosGetProcIO(float *readKB, float *writeKB) { - static int64_t lastReadbyte = -1; - static int64_t lastWritebyte = -1; - - int64_t curReadbyte = 0; - int64_t curWritebyte = 0; - - if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { - return false; - } - - if (lastReadbyte == -1 || lastWritebyte == -1) { - lastReadbyte = curReadbyte; - lastWritebyte = curWritebyte; - return false; - } - - *readKB = (float)((double)(curReadbyte - lastReadbyte) / 1024); - *writeKB = (float)((double)(curWritebyte - lastWritebyte) / 1024); - if (*readKB < 0) *readKB = 0; - if (*writeKB < 0) *writeKB = 0; - - lastReadbyte = curReadbyte; - lastWritebyte = curWritebyte; - - return true; + return -1; } void taosGetSystemInfo() { - tsNumOfCores = taosGetCpuCores(); - tsTotalMemoryMB = taosGetTotalMemory(); + taosGetCpuCores(&tsNumOfCores); + taosGetTotalMemory(&tsTotalMemoryKB); - float tmp1, tmp2; - taosGetBandSpeed(&tmp1); + double tmp1, tmp2, tmp3, tmp4; + taosGetBandSpeed(&tmp1, &tmp2); taosGetCpuUsage(&tmp1, &tmp2); - taosGetProcIO(&tmp1, &tmp2); + taosGetIOSpeed(&tmp1, &tmp2, &tmp3, &tmp4); } void taosKillSystem() { - //printf("function taosKillSystem, exit!"); + // printf("function taosKillSystem, exit!"); exit(0); } int taosSystem(const char *cmd) { - //printf("taosSystem not support"); + // printf("taosSystem not support"); return -1; } @@ -241,66 +209,56 @@ char *taosGetCmdlineByPID(int pid) { return ""; } #include #include - void taosKillSystem() { - //printf("function taosKillSystem, exit!"); + // printf("function taosKillSystem, exit!"); exit(0); } -int32_t taosGetCpuCores() { return sysconf(_SC_NPROCESSORS_ONLN); } +int32_t taosGetCpuCores(float *numOfCores) { + *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); + return 0; +} void taosGetSystemInfo() { - // taosGetProcInfos(); - - tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); long physical_pages = sysconf(_SC_PHYS_PAGES); long page_size = sysconf(_SC_PAGESIZE); - tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); - tsPageSize = page_size; + tsTotalMemoryKB = physical_pages * page_size / 1024; + tsPageSizeKB = page_size / 1024; + tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); } -bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { +int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { if (rchars) *rchars = 0; if (wchars) *wchars = 0; - return true; + if (read_bytes) *read_bytes = 0; + if (write_bytes) *write_bytes = 0; + return 0; } -bool taosGetProcIO(float *readKB, float *writeKB) { - *readKB = 0; - *writeKB = 0; - return true; +int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { + *receive_bytes = 0; + *transmit_bytes = 0; + return 0; } -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { - if (bytes) *bytes = 0; - if (rbytes) *rbytes = 0; - if (tbytes) *tbytes = 0; - return true; -} - -bool taosGetBandSpeed(float *bandSpeedKb) { - *bandSpeedKb = 0; - return true; -} - -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { +int32_t taosGetCpuUsage(double *sysCpuUsage, double *procCpuUsage) { *sysCpuUsage = 0; *procCpuUsage = 0; - return true; + return 0; } -bool taosGetProcMemory(float *memoryUsedMB) { - *memoryUsedMB = 0; - return true; +int32_t taosGetProcMemory(int64_t *usedKB) { + *usedKB = 0; + return 0; } -bool taosGetSysMemory(float *memoryUsedMB) { - *memoryUsedMB = 0; - return true; +int32_t taosGetSysMemory(int64_t *usedKB) { + *usedKB = 0; + return 0; } int taosSystem(const char *cmd) { - //printf("un support funtion"); + // printf("un support funtion"); return -1; } @@ -309,7 +267,7 @@ void taosSetCoreDump() {} int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { struct statvfs info; if (statvfs(dataDir, &info)) { - //printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } else { @@ -377,39 +335,37 @@ static char tsSysCpuFile[] = "/proc/stat"; static char tsProcCpuFile[25] = {0}; static char tsProcMemFile[25] = {0}; static char tsProcIOFile[25] = {0}; -static float tsPageSizeKB = 0; static void taosGetProcInfos() { - tsPageSize = sysconf(_SC_PAGESIZE); + tsPageSizeKB = sysconf(_SC_PAGESIZE) / 1024; tsOpenMax = sysconf(_SC_OPEN_MAX); tsStreamMax = sysconf(_SC_STREAM_MAX); - tsProcId = (pid_t)syscall(SYS_gettid); - tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; - snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); - snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); - snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); + snprintf(tsProcMemFile, sizeof(tsProcMemFile), "/proc/%d/status", tsProcId); + snprintf(tsProcCpuFile, sizeof(tsProcCpuFile), "/proc/%d/stat", tsProcId); + snprintf(tsProcIOFile, sizeof(tsProcIOFile), "/proc/%d/io", tsProcId); } -static int32_t taosGetTotalMemory() { return (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); } - -bool taosGetSysMemory(float *memoryUsedMB) { - float memoryAvailMB = (float)sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB / 1024; - *memoryUsedMB = (float)tsTotalMemoryMB - memoryAvailMB; - return true; +int32_t taosGetTotalMemory(int64_t *totalKB) { + *totalKB = (int64_t)(sysconf(_SC_PHYS_PAGES) * tsPageSizeKB); + return 0; } -bool taosGetProcMemory(float *memoryUsedMB) { - // FILE *fp = fopen(tsProcMemFile, "r"); +int32_t taosGetSysMemory(int64_t *usedKB) { + *usedKB = sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB; + return 0; +} + +int32_t taosGetProcMemory(int64_t *usedKB) { TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - //printf("open file:%s failed", tsProcMemFile); - return false; + // printf("open file:%s failed", tsProcMemFile); + return -1; } ssize_t _bytes = 0; - char * line = NULL; + char *line = NULL; while (!taosEOFFile(pFile)) { _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { @@ -421,60 +377,56 @@ bool taosGetProcMemory(float *memoryUsedMB) { } if (line == NULL) { - //printf("read file:%s failed", tsProcMemFile); + // printf("read file:%s failed", tsProcMemFile); taosCloseFile(&pFile); - return false; + return -1; } - int64_t memKB = 0; - char tmp[10]; - sscanf(line, "%s %" PRId64, tmp, &memKB); - *memoryUsedMB = (float)((double)memKB / 1024); + char tmp[10]; + sscanf(line, "%s %" PRId64, tmp, usedKB); - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); - return true; + return 0; } -static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { - // FILE *fp = fopen(tsSysCpuFile, "r"); +static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - //printf("open file:%s failed", tsSysCpuFile); - return false; + // printf("open file:%s failed", tsSysCpuFile); + return -1; } - char * line = NULL; + char *line = NULL; ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { - //printf("read file:%s failed", tsSysCpuFile); + // printf("read file:%s failed", tsSysCpuFile); taosCloseFile(&pFile); - return false; + return -1; } char cpu[10] = {0}; sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); - return true; + return 0; } -static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { - // FILE *fp = fopen(tsProcCpuFile, "r"); +static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - //printf("open file:%s failed", tsProcCpuFile); - return false; + // printf("open file:%s failed", tsProcCpuFile); + return -1; } - char * line = NULL; + char *line = NULL; ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { - //printf("read file:%s failed", tsProcCpuFile); + // printf("read file:%s failed", tsProcCpuFile); taosCloseFile(&pFile); - return false; + return -1; } for (int i = 0, blank = 0; line[i] != 0; ++i) { @@ -486,26 +438,28 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); - return true; + return 0; } +int32_t taosGetCpuCores(float *numOfCores) { + *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); + return 0; +} -int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } - -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { +int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine) { static uint64_t lastSysUsed = 0; static uint64_t lastSysTotal = 0; static uint64_t lastProcTotal = 0; SysCpuInfo sysCpu; ProcCpuInfo procCpu; - if (!taosGetSysCpuInfo(&sysCpu)) { - return false; + if (taosGetSysCpuInfo(&sysCpu) != 0) { + return -1; } - if (!taosGetProcCpuInfo(&procCpu)) { - return false; + if (taosGetProcCpuInfo(&procCpu) != 0) { + return -1; } uint64_t curSysUsed = sysCpu.user + sysCpu.nice + sysCpu.system; @@ -516,21 +470,21 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { lastSysUsed = curSysUsed > 1 ? curSysUsed : 1; lastSysTotal = curSysTotal > 1 ? curSysTotal : 1; lastProcTotal = curProcTotal > 1 ? curProcTotal : 1; - return false; + return -1; } if (curSysTotal == lastSysTotal) { - return false; + return -1; } - *sysCpuUsage = (float)((double)(curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100); - *procCpuUsage = (float)((double)(curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100); + *cpu_engine = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; + *cpu_system = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100; lastSysUsed = curSysUsed; lastSysTotal = curSysTotal; lastProcTotal = curProcTotal; - return true; + return 0; } int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { @@ -540,22 +494,17 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { } else { diskSize->total = info.f_blocks * info.f_frsize; diskSize->avail = info.f_bavail * info.f_frsize; - diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; + diskSize->used = diskSize->total - diskSize->avail; return 0; } } -bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { - *bytes = 0; - // FILE *fp = fopen(tsSysNetFile, "r"); +int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - //printf("open file:%s failed", tsSysNetFile); - return false; - } + if (pFile == NULL) return -1; ssize_t _bytes = 0; - char * line = NULL; + char *line = NULL; while (!taosEOFFile(pFile)) { int64_t o_rbytes = 0; @@ -585,69 +534,62 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64, nouse0, &o_rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &o_tbytes, &tpackets); - if (rbytes) *rbytes = o_rbytes; - if (tbytes) *tbytes = o_tbytes; - *bytes += (o_rbytes + o_tbytes); + *receive_bytes = o_rbytes; + *transmit_bytes = o_tbytes; } - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); - return true; + return 0; } -bool taosGetBandSpeed(float *bandSpeedKb) { - static int64_t lastBytes = 0; - static time_t lastTime = 0; - int64_t curBytes = 0; - time_t curTime = time(NULL); +int32_t taosGetBandSpeed(double *receive_bytes_per_sec, double *transmit_bytes_per_sec) { + static int64_t last_receive_bytes = 0; + static int64_t last_transmit_bytes = 0; + static int64_t last_time = 0; + int64_t cur_receive_bytes = 0; + int64_t cur_transmit_bytes = 0; + int64_t cur_time = taosGetTimestampMs(); - if (!taosGetCardInfo(&curBytes, NULL, NULL)) { - return false; + if (taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes) != 0) { + return -1; } - if (lastTime == 0 || lastBytes == 0) { - lastTime = curTime; - lastBytes = curBytes; - *bandSpeedKb = 0; - return true; + if (last_time == 0 || last_time >= cur_time) { + last_time = cur_time; + last_receive_bytes = cur_receive_bytes; + last_transmit_bytes = cur_transmit_bytes; + *receive_bytes_per_sec = 0; + *transmit_bytes_per_sec = 0; + return 0; } - if (lastTime >= curTime || lastBytes > curBytes) { - lastTime = curTime; - lastBytes = curBytes; - *bandSpeedKb = 0; - return true; - } + *receive_bytes_per_sec = (cur_receive_bytes - last_receive_bytes) / (double)(cur_time - last_time) * 1000; + *transmit_bytes_per_sec = (cur_transmit_bytes - last_transmit_bytes) / (double)(cur_time - last_time) * 1000; - double totalBytes = (double)(curBytes - lastBytes) / 1024 * 8; // Kb - *bandSpeedKb = (float)(totalBytes / (double)(curTime - lastTime)); + last_time = cur_time; + last_transmit_bytes = cur_transmit_bytes; + last_receive_bytes = cur_receive_bytes; - // //printf("bandwidth lastBytes:%ld, lastTime:%ld, curBytes:%ld, curTime:%ld, - // speed:%f", lastBytes, lastTime, curBytes, curTime, *bandSpeed); + if (*receive_bytes_per_sec < 0) *receive_bytes_per_sec = 0; + if (*transmit_bytes_per_sec < 0) *transmit_bytes_per_sec = 0; - lastTime = curTime; - lastBytes = curBytes; - - return true; + return 0; } -bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { - // FILE *fp = fopen(tsProcIOFile, "r"); +int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - //printf("open file:%s failed", tsProcIOFile); - return false; - } + if (pFile == NULL) return -1; ssize_t _bytes = 0; - char * line = NULL; - char tmp[10]; + char *line = NULL; + char tmp[24]; int readIndex = 0; while (!taosEOFFile(pFile)) { _bytes = taosGetLineFile(pFile, &line); - if ((_bytes < 0) || (line == NULL)) { + if (_bytes < 10 || line == NULL) { break; } if (strstr(line, "rchar:") != NULL) { @@ -656,69 +598,88 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { } else if (strstr(line, "wchar:") != NULL) { sscanf(line, "%s %" PRId64, tmp, wchars); readIndex++; + } else if (strstr(line, "read_bytes:") != NULL) { // read_bytes + sscanf(line, "%s %" PRId64, tmp, read_bytes); + readIndex++; + } else if (strstr(line, "write_bytes:") != NULL) { // write_bytes + sscanf(line, "%s %" PRId64, tmp, write_bytes); + readIndex++; } else { } - if (readIndex >= 2) break; + if (readIndex >= 4) break; } - if(line != NULL) tfree(line); + if (line != NULL) tfree(line); taosCloseFile(&pFile); - if (readIndex < 2) { - //printf("read file:%s failed", tsProcIOFile); - return false; + if (readIndex < 4) { + return -1; } - return true; + return 0; } -bool taosGetProcIO(float *readKB, float *writeKB) { - static int64_t lastReadbyte = -1; - static int64_t lastWritebyte = -1; +int32_t taosGetIOSpeed(double *rchar_per_sec, double *wchar_per_sec, double *read_bytes_per_sec, + double *write_bytes_per_sec) { + static int64_t last_rchar = -1; + static int64_t last_wchar = -1; + static int64_t last_read_bytes = -1; + static int64_t last_write_bytes = -1; + static int64_t last_time = 0; - int64_t curReadbyte = 0; - int64_t curWritebyte = 0; + int64_t cur_rchar = 0; + int64_t cur_wchar = 0; + int64_t cur_read_bytes = 0; + int64_t cur_write_bytes = 0; + int64_t cur_time = taosGetTimestampMs(); - if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { - return false; + if (taosReadProcIO(&cur_rchar, &cur_wchar, &cur_read_bytes, &cur_write_bytes) != 0) { + return -1; } - if (lastReadbyte == -1 || lastWritebyte == -1) { - lastReadbyte = curReadbyte; - lastWritebyte = curWritebyte; - return false; + if (last_time == 0 || last_time >= cur_time) { + last_time = cur_time; + last_rchar = cur_rchar; + last_wchar = cur_wchar; + last_read_bytes = cur_read_bytes; + last_write_bytes = cur_write_bytes; + return -1; } - *readKB = (float)((double)(curReadbyte - lastReadbyte) / 1024); - *writeKB = (float)((double)(curWritebyte - lastWritebyte) / 1024); - if (*readKB < 0) *readKB = 0; - if (*writeKB < 0) *writeKB = 0; + *rchar_per_sec = (cur_rchar - last_rchar) / (double)(cur_time - last_time) * 1000; + *wchar_per_sec = (cur_wchar - last_wchar) / (double)(cur_time - last_time) * 1000; + *read_bytes_per_sec = (cur_read_bytes - last_read_bytes) / (double)(cur_time - last_time) * 1000; + *write_bytes_per_sec = (cur_write_bytes - last_write_bytes) / (double)(cur_time - last_time) * 1000; - lastReadbyte = curReadbyte; - lastWritebyte = curWritebyte; + last_time = cur_time; + last_rchar = cur_rchar; + last_wchar = cur_wchar; + last_read_bytes = cur_read_bytes; + last_write_bytes = cur_write_bytes; - return true; + if (*rchar_per_sec < 0) *rchar_per_sec = 0; + if (*wchar_per_sec < 0) *wchar_per_sec = 0; + if (*read_bytes_per_sec < 0) *read_bytes_per_sec = 0; + if (*write_bytes_per_sec < 0) *write_bytes_per_sec = 0; + + return 0; } void taosGetSystemInfo() { taosGetProcInfos(); + taosGetCpuCores(&tsNumOfCores); + taosGetTotalMemory(&tsTotalMemoryKB); - tsNumOfCores = taosGetCpuCores(); - tsTotalMemoryMB = taosGetTotalMemory(); - - float tmp1, tmp2; - taosGetSysMemory(&tmp1); - taosGetProcMemory(&tmp2); - taosGetBandSpeed(&tmp1); + double tmp1, tmp2, tmp3, tmp4; + taosGetBandSpeed(&tmp1, &tmp2); taosGetCpuUsage(&tmp1, &tmp2); - taosGetProcIO(&tmp1, &tmp2); - + taosGetIOSpeed(&tmp1, &tmp2, &tmp3, &tmp4); } void taosKillSystem() { // SIGINT - //printf("taosd will shut down soon"); + // printf("taosd will shut down soon"); kill(tsProcId, 2); } @@ -727,22 +688,22 @@ int taosSystem(const char *cmd) { int res; char buf[1024]; if (cmd == NULL) { - //printf("taosSystem cmd is NULL!"); + // printf("taosSystem cmd is NULL!"); return -1; } if ((fp = popen(cmd, "r")) == NULL) { - //printf("popen cmd:%s error: %s", cmd, strerror(errno)); + // printf("popen cmd:%s error: %s", cmd, strerror(errno)); return -1; } else { while (fgets(buf, sizeof(buf), fp)) { - //printf("popen result:%s", buf); + // printf("popen result:%s", buf); } if ((res = pclose(fp)) == -1) { - //printf("close popen file pointer fp error!"); + // printf("close popen file pointer fp error!"); } else { - //printf("popen res is :%d", res); + // printf("popen res is :%d", res); } return res; @@ -757,14 +718,14 @@ void taosSetCoreDump(bool enable) { struct rlimit rlim_new; if (getrlimit(RLIMIT_CORE, &rlim) == 0) { #ifndef _ALPINE - //printf("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); + // printf("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); #else - //printf("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); + // printf("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); #endif rlim_new.rlim_cur = RLIM_INFINITY; rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { - //printf("set unlimited fail, error: %s", strerror(errno)); + // printf("set unlimited fail, error: %s", strerror(errno)); rlim_new.rlim_cur = rlim.rlim_max; rlim_new.rlim_max = rlim.rlim_max; (void)setrlimit(RLIMIT_CORE, &rlim_new); @@ -773,9 +734,9 @@ void taosSetCoreDump(bool enable) { if (getrlimit(RLIMIT_CORE, &rlim) == 0) { #ifndef _ALPINE - //printf("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); + // printf("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); #else - //printf("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); + // printf("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); #endif } @@ -801,10 +762,10 @@ void taosSetCoreDump(bool enable) { old_len = sizeof(old_usespid); if (syscall(SYS__sysctl, &args) == -1) { - //printf("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); + // printf("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); } - //printf("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); + // printf("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); old_usespid = 0; old_len = 0; @@ -817,10 +778,10 @@ void taosSetCoreDump(bool enable) { old_len = sizeof(old_usespid); if (syscall(SYS__sysctl, &args) == -1) { - //printf("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); + // printf("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); } - //printf("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); + // printf("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); #endif } @@ -885,7 +846,7 @@ SysNameInfo taosGetSysNameInfo() { return info; } -bool taosGetEmail(char *email, int32_t maxLen) { +int32_t taosGetEmail(char *email, int32_t maxLen) { const char *filepath = "/usr/local/taos/email"; TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); @@ -893,17 +854,17 @@ bool taosGetEmail(char *email, int32_t maxLen) { if (taosReadFile(pFile, (void *)email, maxLen) < 0) { taosCloseFile(&pFile); - return false; + return -1; } taosCloseFile(&pFile); - return true; + return 0; } -bool taosGetOsReleaseName(char *releaseName, int32_t maxLen) { - char *line = NULL; - size_t size = 0; - bool ret = false; +int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { + char *line = NULL; + size_t size = 0; + int32_t code = -1; TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return false; @@ -917,21 +878,21 @@ bool taosGetOsReleaseName(char *releaseName, int32_t maxLen) { line[size - 2] = 0; } tstrncpy(releaseName, p, maxLen); - ret = true; + code = 0; break; } } if (line != NULL) free(line); taosCloseFile(&pFile); - return ret; + return code; } -bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { +int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { char *line = NULL; size_t size = 0; int32_t done = 0; - bool ret = false; + int32_t code = -1; TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return false; @@ -941,11 +902,11 @@ bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { const char *v = strchr(line, ':') + 2; tstrncpy(cpuModel, v, maxLen); - ret = true; + code = 0; done |= 1; } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { const char *v = strchr(line, ':') + 2; - *numOfCores = atoi(v); + *numOfCores = atof(v); done |= 2; } } @@ -953,31 +914,7 @@ bool taosGetCpuInfo(char *cpuModel, int32_t maxLen, int32_t *numOfCores) { if (line != NULL) free(line); taosCloseFile(&pFile); - return ret; -} - -bool taosGetTotalSysMemoryKB(uint64_t *kb) { - char *line = NULL; - size_t size = 0; - bool ret = false; - - TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; - - while ((size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "MemTotal", 8) == 0) { - const char *p = strchr(line, ':') + 1; - while (*p == ' ') p++; - ret = true; - *kb = atoll(p); - break; - } - } - - if (line != NULL) free(line); - taosCloseFile(&pFile); - return ret; + return code; } #endif diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 64d822f750..51ea2b1f13 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -15,11 +15,99 @@ #define _DEFAULT_SOURCE #include "tcache.h" +#include "taoserror.h" #include "tlog.h" -#include "ttimer.h" #include "tutil.h" -static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) { +#define CACHE_MAX_CAPACITY 1024*1024*16 +#define CACHE_DEFAULT_CAPACITY 1024*4 + +static pthread_t cacheRefreshWorker = {0}; +static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; +static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; +static SArray *pCacheArrayList = NULL; +static bool stopRefreshWorker = false; +static bool refreshWorkerNormalStopped = false; +static bool refreshWorkerUnexpectedStopped = false; + +typedef struct SCacheNode { + uint64_t addedTime; // the added time when this element is added or updated into cache + uint64_t lifespan; // life duration when this element should be remove from cache + int64_t expireTime; // expire time + uint64_t signature; + struct STrashElem *pTNodeHeader; // point to trash node head + uint16_t keyLen: 15; // max key size: 32kb + bool inTrashcan : 1; // denote if it is in trash or not + uint32_t size; // allocated size for current SCacheNode + uint32_t dataLen; + T_REF_DECLARE() + struct SCacheNode *pNext; + char *key; + char *data; +} SCacheNode; + +typedef struct SCacheEntry { + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SCacheNode *next; +} SCacheEntry; + +typedef struct STrashElem { + struct STrashElem *prev; + struct STrashElem *next; + SCacheNode *pData; +} STrashElem; + +typedef struct SCacheIter { + SCacheObj *pCacheObj; + SCacheNode **pCurrent; + int32_t entryIndex; + int32_t index; + int32_t numOfObj; +} SCacheIter; + +/* + * to accommodate the old data which has the same key value of new one in hashList + * when an new node is put into cache, if an existed one with the same key: + * 1. if the old one does not be referenced, update it. + * 2. otherwise, move the old one to pTrash, addedTime the new one. + * + * when the node in pTrash does not be referenced, it will be release at the expired expiredTime + */ +struct SCacheObj { + int64_t sizeInBytes; // total allocated buffer in this hash table, SCacheObj is not included. + int64_t refreshTime; + char *name; + SCacheStatis statistics; + + SCacheEntry *pEntryList; + size_t capacity; // number of slots + size_t numOfElems; // number of elements in cache + _hash_fn_t hashFp; // hash function + __cache_free_fn_t freeFp; + + uint32_t numOfElemsInTrash; // number of element in trash + STrashElem *pTrash; + + uint8_t deleting; // set the deleting flag to stop refreshing ASAP. + pthread_t refreshWorker; + bool extendLifespan; // auto extend life span when one item is accessed. + int64_t checkTick; // tick used to record the check times of the refresh threads +#if defined(LINUX) + pthread_rwlock_t lock; +#else + pthread_mutex_t lock; +#endif +}; + +typedef struct SCacheObjTravSup { + SCacheObj *pCacheObj; + int64_t time; + __cache_trav_fn_t fp; + void *param1; +} SCacheObjTravSup; + +static FORCE_INLINE void __trashcan_wr_lock(SCacheObj *pCacheObj) { #if defined(LINUX) pthread_rwlock_wrlock(&pCacheObj->lock); #else @@ -27,7 +115,7 @@ static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) { #endif } -static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) { +static FORCE_INLINE void __trashcan_unlock(SCacheObj *pCacheObj) { #if defined(LINUX) pthread_rwlock_unlock(&pCacheObj->lock); #else @@ -35,7 +123,7 @@ static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) { #endif } -static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pCacheObj) { +static FORCE_INLINE int32_t __trashcan_lock_init(SCacheObj *pCacheObj) { #if defined(LINUX) return pthread_rwlock_init(&pCacheObj->lock, NULL); #else @@ -43,7 +131,7 @@ static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pCacheObj) { #endif } -static FORCE_INLINE void __cache_lock_destroy(SCacheObj *pCacheObj) { +static FORCE_INLINE void __trashcan_lock_destroy(SCacheObj *pCacheObj) { #if defined(LINUX) pthread_rwlock_destroy(&pCacheObj->lock); #else @@ -63,14 +151,6 @@ static void doCleanupDataCache(SCacheObj *pCacheObj); */ static void *taosCacheTimedRefresh(void *handle); -static pthread_t cacheRefreshWorker = {0}; -static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; -static SArray *pCacheArrayList = NULL; -static bool stopRefreshWorker = false; -static bool refreshWorkerNormalStopped = false; -static bool refreshWorkerUnexpectedStopped = false; - static void doInitRefreshThread(void) { pCacheArrayList = taosArrayInit(4, POINTER_BYTES); @@ -99,9 +179,9 @@ pthread_t doRegisterCacheObj(SCacheObj *pCacheObj) { * in pData. Pointer copy causes memory access error. * @param size size of block * @param lifespan total survial expiredTime from now - * @return SCacheDataNode + * @return SCacheNode */ -static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, +static SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration); /** @@ -110,7 +190,7 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const * @param pCacheObj Cache object * @param pNode Cache slot object */ -static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode); +static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode); /** * remove nodes in trash with refCount == 0 in cache @@ -126,18 +206,16 @@ static void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force); * @param pCacheObj cache object * @param pNode data node */ -static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNode *pNode) { +static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode *pNode) { if (pNode->signature != (uint64_t)pNode) { uError("key:%s, %p data is invalid, or has been released", pNode->key, pNode); return; } - atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); - int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable); - assert(size > 0); + atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, total num:%d size:%" PRId64 "bytes", - pCacheObj->name, pNode->key, pNode->data, pNode->size, size - 1, pCacheObj->totalSize); + pCacheObj->name, pNode->key, pNode->data, pNode->size, (int)pCacheObj->numOfElems - 1, pCacheObj->sizeInBytes); if (pCacheObj->freeFp) { pCacheObj->freeFp(pNode->data); @@ -181,6 +259,97 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem free(pElem); } +static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { + assert(pNode != NULL && pEntry != NULL); + + pNode->pNext = pEntry->next; + pEntry->next = pNode; + pEntry->num += 1; +} + +static void removeNodeInEntryList(SCacheEntry* pe, SCacheNode* prev, SCacheNode* pNode) { + if (prev == NULL) { + ASSERT(pe->next == pNode); + pe->next = pNode->pNext; + } else { + prev->pNext = pNode->pNext; + } + + pNode->pNext = NULL; + pe->num -= 1; +} + +static FORCE_INLINE SCacheEntry* doFindEntry(SCacheObj* pCacheObj, const void* key, size_t keyLen) { + uint32_t hashVal = (*pCacheObj->hashFp)(key, keyLen); + int32_t slot = hashVal % pCacheObj->capacity; + return &pCacheObj->pEntryList[slot]; +} + +static FORCE_INLINE SCacheNode * +doSearchInEntryList(SCacheEntry *pe, const void *key, size_t keyLen, SCacheNode** prev) { + SCacheNode *pNode = pe->next; + while (pNode) { + if ((pNode->keyLen == keyLen) && memcmp(pNode->key, key, keyLen) == 0) { + break; + } + *prev = pNode; + pNode = pNode->pNext; + } + + return pNode; +} + +static bool doRemoveExpiredFn(void *param, SCacheNode* pNode) { + SCacheObjTravSup *ps = (SCacheObjTravSup *)param; + SCacheObj *pCacheObj = ps->pCacheObj; + + if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { + taosCacheReleaseNode(pCacheObj, pNode); + + // this node should be remove from hash table + return false; + } + + if (ps->fp) { + (ps->fp)(pNode->data, ps->param1); + } + + // do not remove element in hash table + return true; +} + +static bool doRemoveNodeFn(void *param, SCacheNode *pNode) { + SCacheObjTravSup *ps = (SCacheObjTravSup *)param; + SCacheObj *pCacheObj = ps->pCacheObj; + + if (T_REF_VAL_GET(pNode) == 0) { + taosCacheReleaseNode(pCacheObj, pNode); + } else { // do add to trashcan + taosAddToTrashcan(pCacheObj, pNode); + } + + // this node should be remove from hash table + return false; +} + +static FORCE_INLINE int32_t getCacheCapacity(int32_t length) { + int32_t len = 0; + if (length < CACHE_DEFAULT_CAPACITY) { + len = CACHE_DEFAULT_CAPACITY; + return len; + } else if (length > CACHE_MAX_CAPACITY) { + len = CACHE_MAX_CAPACITY; + return len; + } + + len = CACHE_DEFAULT_CAPACITY; + while (len < length && len < CACHE_MAX_CAPACITY) { + len = (len << 1u); + } + + return len > CACHE_MAX_CAPACITY? CACHE_MAX_CAPACITY:len; +} + SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char *cacheName) { const int32_t SLEEP_DURATION = 500; // 500 ms @@ -195,39 +364,42 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pHashTable = taosHashInit(4096, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK); - pCacheObj->name = strdup(cacheName); - if (pCacheObj->pHashTable == NULL) { + // TODO add the auto extend procedure + pCacheObj->capacity = 4096; + pCacheObj->pEntryList = calloc(pCacheObj->capacity, sizeof(SCacheEntry)); + if (pCacheObj->pEntryList == NULL) { free(pCacheObj); uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } // set free cache node callback function - pCacheObj->freeFp = fn; - pCacheObj->refreshTime = refreshTimeInSeconds * 1000; - pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; + pCacheObj->hashFp = taosGetDefaultHashFunction(keyType); + pCacheObj->freeFp = fn; + pCacheObj->refreshTime = refreshTimeInSeconds * 1000; + pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; pCacheObj->extendLifespan = extendLifespan; // the TTL after the last access - if (__cache_lock_init(pCacheObj) != 0) { - taosHashCleanup(pCacheObj->pHashTable); + if (__trashcan_lock_init(pCacheObj) != 0) { + tfree(pCacheObj->pEntryList); free(pCacheObj); uError("failed to init lock, reason:%s", strerror(errno)); return NULL; } + pCacheObj->name = strdup(cacheName); doRegisterCacheObj(pCacheObj); return pCacheObj; } void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int32_t durationMS) { - if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) { + if (pCacheObj == NULL || pCacheObj->pEntryList == NULL || pCacheObj->deleting == 1) { return NULL; } - SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS); + SCacheNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS); if (pNode1 == NULL) { uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); return NULL; @@ -235,86 +407,77 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v T_REF_INC(pNode1); - int32_t succ = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); - if (succ == 0) { - atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); + SCacheEntry *pe = doFindEntry(pCacheObj, key, keyLen); + + taosWLockLatch(&pe->latch); + + SCacheNode *prev = NULL; + SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev); + + if (pNode == NULL) { + pushfrontNodeInEntryList(pe, pNode1); + atomic_add_fetch_64(&pCacheObj->numOfElems, 1); + atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size); uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 - ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes", - pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, - (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); + ", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes", + pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems, + pCacheObj->sizeInBytes, (int64_t)dataSize); } else { // duplicated key exists - while (1) { - SCacheDataNode *p = NULL; - // int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); - int32_t ret = taosHashRemove(pCacheObj->pHashTable, key, keyLen); + // move current node to trashcan + removeNodeInEntryList(pe, prev, pNode); - // add to trashcan - if (ret == 0) { - if (T_REF_VAL_GET(p) == 0) { - if (pCacheObj->freeFp) { - pCacheObj->freeFp(p->data); - } - - atomic_sub_fetch_64(&pCacheObj->totalSize, p->size); - tfree(p); - } else { - taosAddToTrashcan(pCacheObj, p); - uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data); - } + if (T_REF_VAL_GET(pNode) == 0) { + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pNode->data); } - assert(T_REF_VAL_GET(pNode1) == 1); - - ret = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); - if (ret == 0) { - atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); - - uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 - ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes", - pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, - (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); - - return pNode1->data; - - } else { - // failed, try again - } + atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); + tfree(pNode); + } else { + taosAddToTrashcan(pCacheObj, pNode); + uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pNode->data); } + + pushfrontNodeInEntryList(pe, pNode1); + atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size); + uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 + ", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes", + pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems, + pCacheObj->sizeInBytes, (int64_t)dataSize); } + taosWUnLockLatch(&pe->latch); return pNode1->data; } -static void incRefFn(void *ptNode) { - assert(ptNode != NULL); - - SCacheDataNode **p = (SCacheDataNode **)ptNode; - assert(T_REF_VAL_GET(*p) >= 0); - - int32_t ret = T_REF_INC(*p); - assert(ret > 0); -} - void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) { if (pCacheObj == NULL || pCacheObj->deleting == 1) { return NULL; } - if (taosHashGetSize(pCacheObj->pHashTable) == 0) { + if (pCacheObj->numOfElems == 0) { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); return NULL; } - SCacheDataNode *ptNode = NULL; - taosHashGetClone(pCacheObj->pHashTable, key, keyLen, &ptNode); - // taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode); + SCacheNode *prev = NULL; + SCacheEntry *pe = doFindEntry(pCacheObj, key, keyLen); - void *pData = (ptNode != NULL) ? ptNode->data : NULL; + taosRLockLatch(&pe->latch); + SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev); + if (pNode != NULL) { + int32_t ref = T_REF_INC(pNode); + ASSERT(ref > 0); + } + + taosRUnLockLatch(&pe->latch); + + void *pData = (pNode != NULL) ? pNode->data : NULL; if (pData != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, - T_REF_VAL_GET(ptNode)); + T_REF_VAL_GET(pNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); @@ -327,9 +490,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { if (pCacheObj == NULL || data == NULL) return NULL; - size_t offset = offsetof(SCacheDataNode, data); - SCacheDataNode *ptNode = (SCacheDataNode *)((char *)data - offset); - + SCacheNode *ptNode = (SCacheNode *)((char *)data - sizeof(SCacheNode)); if (ptNode->signature != (uint64_t)ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; @@ -343,24 +504,20 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { return data; } -void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { +void *taosCacheTransferData(SCacheObj *pCacheObj, void **data) { if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL; - size_t offset = offsetof(SCacheDataNode, data); - SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset); - + SCacheNode *ptNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); if (ptNode->signature != (uint64_t)ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; } assert(T_REF_VAL_GET(ptNode) >= 1); - char *d = *data; // clear its reference to old area *data = NULL; - return d; } @@ -378,9 +535,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // therefore the check for the empty of both the hash table and the trashcan has a race condition. // It happens when there is only one object in the cache, and two threads which has referenced this object // start to free the it simultaneously [TD-1569]. - size_t offset = offsetof(SCacheDataNode, data); - - SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); + SCacheNode *pNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode)); if (pNode->signature != (uint64_t)pNode) { uError("cache:%s, %p, release invalid cache data", pCacheObj->name, pNode); return; @@ -419,9 +574,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // destroyed by refresh worker if decrease ref count before removing it from linked-list. assert(pNode->pTNodeHeader->pData == pNode); - __cache_wr_lock(pCacheObj); + __trashcan_wr_lock(pCacheObj); doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); - __cache_unlock(pCacheObj); + __trashcan_unlock(pCacheObj); ref = T_REF_DEC(pNode); assert(ref == 0); @@ -434,36 +589,37 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } else { // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread // when reaches here. - SCacheDataNode *p = NULL; - int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - // int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, pNode->key, pNode->keySize, &p, sizeof(void - // *)); + SCacheNode * prev = NULL; + SCacheEntry *pe = doFindEntry(pCacheObj, pNode->key, pNode->keyLen); + + taosWLockLatch(&pe->latch); ref = T_REF_DEC(pNode); - // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. - // note that the remove operation can be executed only once. - if (ret == 0) { + SCacheNode *p = doSearchInEntryList(pe, pNode->key, pNode->keyLen, &prev); + + if (p != NULL) { + // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. + // note that the remove operation can be executed only once. if (p != pNode) { uDebug( - "cache:%s, key:%p, successfully removed a new entry:%p, refcnt:%d, prev entry:%p has been removed by " - "others already", - pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data); + "cache:%s, key:%p, a new entry:%p found, refcnt:%d, prev entry:%p, refcnt:%d has been removed by " + "others already, prev must in trashcan", + pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data, T_REF_VAL_GET(pNode)); - assert(p->pTNodeHeader == NULL); - taosAddToTrashcan(pCacheObj, p); + assert(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL); } else { + removeNodeInEntryList(pe, prev, p); uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); if (ref > 0) { assert(pNode->pTNodeHeader == NULL); - taosAddToTrashcan(pCacheObj, pNode); } else { // ref == 0 - atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); + atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); - int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable); + int32_t size = (int32_t)pCacheObj->numOfElems; uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, totalNum:%d size:%" PRId64 "bytes", - pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize); + pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->sizeInBytes); if (pCacheObj->freeFp) { pCacheObj->freeFp(pNode->data); @@ -472,6 +628,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { free(pNode); } } + + taosWUnLockLatch(&pe->latch); } else { uDebug("cache:%s, key:%p, %p has been removed from hash table by others already, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); @@ -483,59 +641,43 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { char *key = pNode->key; char *p = pNode->data; - // int32_t ref = T_REF_VAL_GET(pNode); - // - // if (ref == 1 && inTrashcan) { - // // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may - // be - // // destroyed by refresh worker if decrease ref count before removing it from linked-list. - // assert(pNode->pTNodeHeader->pData == pNode); - // - // __cache_wr_lock(pCacheObj); - // doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); - // __cache_unlock(pCacheObj); - // - // ref = T_REF_DEC(pNode); - // assert(ref == 0); - // - // doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); - // } else { - // ref = T_REF_DEC(pNode); - // assert(ref >= 0); - // } - int32_t ref = T_REF_DEC(pNode); uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan); } } -typedef struct SHashTravSupp { - SCacheObj *pCacheObj; - int64_t time; - __cache_trav_fn_t fp; - void *param1; -} SHashTravSupp; +void doTraverseElems(SCacheObj* pCacheObj, bool (*fp)(void *param, SCacheNode* pNode), SCacheObjTravSup* pSup) { + int32_t numOfEntries = (int32_t)pCacheObj->capacity; + for (int32_t i = 0; i < numOfEntries; ++i) { + SCacheEntry *pEntry = &pCacheObj->pEntryList[i]; + if (pEntry->num == 0) { + continue; + } -static bool travHashTableEmptyFn(void *param, void *data) { - SHashTravSupp *ps = (SHashTravSupp *)param; - SCacheObj *pCacheObj = ps->pCacheObj; + taosWLockLatch(&pEntry->latch); - SCacheDataNode *pNode = *(SCacheDataNode **)data; + SCacheNode *pNode = pEntry->next; + while (pNode != NULL) { + SCacheNode *next = pNode->pNext; - if (T_REF_VAL_GET(pNode) == 0) { - taosCacheReleaseNode(pCacheObj, pNode); - } else { // do add to trashcan - taosAddToTrashcan(pCacheObj, pNode); + if (fp(pSup, pNode)) { + pNode = pNode->pNext; + } else { + pEntry->next = next; + pEntry->num -= 1; + + atomic_sub_fetch_64(&pCacheObj->numOfElems, 1); + pNode = next; + } + } + + taosWUnLockLatch(&pEntry->latch); } - - // this node should be remove from hash table - return false; } -void taosCacheEmpty(SCacheObj *pCacheObj) { - SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; - - // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); +void taosCacheEmpty(SCacheObj* pCacheObj) { + SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + doTraverseElems(pCacheObj, doRemoveNodeFn, &sup); taosTrashcanEmpty(pCacheObj, false); } @@ -558,38 +700,41 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { doCleanupDataCache(pCacheObj); } -SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { - size_t totalSize = size + sizeof(SCacheDataNode) + keyLen; +SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { + size_t sizeInBytes = size + sizeof(SCacheNode) + keyLen; - SCacheDataNode *pNewNode = calloc(1, totalSize); + SCacheNode *pNewNode = calloc(1, sizeInBytes); if (pNewNode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } + pNewNode->data = (char*)pNewNode + sizeof(SCacheNode); + pNewNode->dataLen = size; memcpy(pNewNode->data, pData, size); - pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size; - pNewNode->keySize = (uint16_t)keyLen; + pNewNode->key = (char *)pNewNode + sizeof(SCacheNode) + size; + pNewNode->keyLen = (uint16_t)keyLen; memcpy(pNewNode->key, key, keyLen); - pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); - pNewNode->lifespan = duration; + pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); + pNewNode->lifespan = duration; pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan; - pNewNode->signature = (uint64_t)pNewNode; - pNewNode->size = (uint32_t)totalSize; + pNewNode->signature = (uint64_t)pNewNode; + pNewNode->size = (uint32_t)sizeInBytes; return pNewNode; } -void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { +void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) { if (pNode->inTrashcan) { /* node is already in trash */ assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); return; } - __cache_wr_lock(pCacheObj); + __trashcan_wr_lock(pCacheObj); STrashElem *pElem = calloc(1, sizeof(STrashElem)); pElem->pData = pNode; pElem->prev = NULL; @@ -604,14 +749,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pCacheObj->pTrash = pElem; pCacheObj->numOfElemsInTrash++; - __cache_unlock(pCacheObj); + __trashcan_unlock(pCacheObj); uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash); } void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { - __cache_wr_lock(pCacheObj); + __trashcan_wr_lock(pCacheObj); if (pCacheObj->numOfElemsInTrash == 0) { if (pCacheObj->pTrash != NULL) { @@ -620,7 +765,7 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { pCacheObj->numOfElemsInTrash); } - __cache_unlock(pCacheObj); + __trashcan_unlock(pCacheObj); return; } @@ -645,49 +790,28 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { } } - __cache_unlock(pCacheObj); + __trashcan_unlock(pCacheObj); } void doCleanupDataCache(SCacheObj *pCacheObj) { - // SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; - // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); + SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + doTraverseElems(pCacheObj, doRemoveNodeFn, &sup); // todo memory leak if there are object with refcount greater than 0 in hash table? - taosHashCleanup(pCacheObj->pHashTable); taosTrashcanEmpty(pCacheObj, true); - __cache_lock_destroy(pCacheObj); + __trashcan_lock_destroy(pCacheObj); + tfree(pCacheObj->pEntryList); tfree(pCacheObj->name); - memset(pCacheObj, 0, sizeof(SCacheObj)); free(pCacheObj); } -bool travHashTableFn(void *param, void *data) { - SHashTravSupp *ps = (SHashTravSupp *)param; - SCacheObj *pCacheObj = ps->pCacheObj; - - SCacheDataNode *pNode = *(SCacheDataNode **)data; - if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); - - // this node should be remove from hash table - return false; - } - - if (ps->fp) { - (ps->fp)(pNode->data, ps->param1); - } - - // do not remove element in hash table - return true; -} - static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) { assert(pCacheObj != NULL); - SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1}; - // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); + SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1}; + doTraverseElems(pCacheObj, doRemoveExpiredFn, &sup); } void taosCacheRefreshWorkerUnexpectedStopped(void) { @@ -746,7 +870,7 @@ void *taosCacheTimedRefresh(void *handle) { continue; } - size_t elemInHash = taosHashGetSize(pCacheObj->pHashTable); + size_t elemInHash = pCacheObj->numOfElems; if (elemInHash + pCacheObj->numOfElemsInTrash == 0) { continue; } @@ -784,4 +908,99 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1) doCacheRefresh(pCacheObj, now, fp, param1); } -void taosStopCacheRefreshWorker(void) { stopRefreshWorker = true; } \ No newline at end of file +void taosStopCacheRefreshWorker(void) { + stopRefreshWorker = true; +} + +size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj) { + return pCacheObj->numOfElems + pCacheObj->numOfElemsInTrash; +} + +SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj) { + ASSERT(pCacheObj != NULL); + SCacheIter* pIter = calloc(1, sizeof(SCacheIter)); + pIter->pCacheObj = (SCacheObj*) pCacheObj; + pIter->entryIndex = -1; + pIter->index = -1; + return pIter; +} + +bool taosCacheIterNext(SCacheIter* pIter) { + SCacheObj* pCacheObj = pIter->pCacheObj; + + if (pIter->index + 1 >= pIter->numOfObj) { + if (pIter->entryIndex + 1 >= pCacheObj->capacity) { + return false; + } + + // release the reference for all objects in the snapshot + for(int32_t i = 0; i < pIter->numOfObj; ++i) { + char* p= pIter->pCurrent[i]->data; + taosCacheRelease(pCacheObj, (void**) &p, false); + pIter->pCurrent[i] = NULL; + } + + while (1) { + pIter->entryIndex++; + if (pIter->entryIndex >= pCacheObj->capacity) { + return false; + } + + SCacheEntry *pEntry = &pCacheObj->pEntryList[pIter->entryIndex]; + taosRLockLatch(&pEntry->latch); + + if (pEntry->num == 0) { + taosRUnLockLatch(&pEntry->latch); + continue; + } + + if (pIter->numOfObj < pEntry->num) { + char *tmp = realloc(pIter->pCurrent, pEntry->num * POINTER_BYTES); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosRUnLockLatch(&pEntry->latch); + return false; + } + + pIter->pCurrent = (SCacheNode **)tmp; + } + + SCacheNode* pNode = pEntry->next; + for (int32_t i = 0; i < pEntry->num; ++i) { + ASSERT(pNode != NULL); + + pIter->pCurrent[i] = pNode; + int32_t ref = T_REF_INC(pIter->pCurrent[i]); + ASSERT(ref >= 1); + + pNode = pNode->pNext; + } + + pIter->numOfObj = pEntry->num; + taosRUnLockLatch(&pEntry->latch); + + pIter->index = -1; + break; + } + } + + pIter->index += 1; + return true; +} + +void* taosCacheIterGetData(const SCacheIter* pIter, size_t* len) { + SCacheNode* pNode = pIter->pCurrent[pIter->index]; + *len = pNode->dataLen; + return pNode->data; +} + +void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* len) { + SCacheNode* pNode = pIter->pCurrent[pIter->index]; + *len = pNode->keyLen; + return pNode->key; +} + +void taosCacheDestroyIter(SCacheIter* pIter) { + tfree(pIter->pCurrent); + tfree(pIter); +} \ No newline at end of file diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 0f22d15cd4..d4bcc27f60 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -145,14 +145,7 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } -#if 0 - if (taosMkDir(fullDir) != 0) { - uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); - return -1; - } -#endif - - cfgFreeItem(pItem); + tfree(pItem->str); pItem->str = strdup(fullDir); if (pItem->str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -647,7 +640,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0) { + if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 176290ba38..9c2fef5b47 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +// clang-format off + #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" @@ -408,6 +410,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGTYPE, "Invalid msg type") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Unexpected generic error in wal") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR, "tfs out of memory") diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 219fc739ca..8b437b9797 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -15,69 +15,136 @@ #define _DEFAULT_SOURCE #include "thash.h" -#include "tdef.h" +#include "taoserror.h" +#include "os.h" #include "tlog.h" // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT -#define MAX_WARNING_REF_COUNT 10000 -#define EXT_SIZE 1024 -#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) +#define MAX_WARNING_REF_COUNT 10000 +#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_DEFAULT_LOAD_FACTOR (0.75) +#define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define DO_FREE_HASH_NODE(_n) \ - do { \ - tfree(_n); \ - } while (0) +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) -#define FREE_HASH_NODE(_h, _n) \ - do { \ - if ((_h)->freeFp) { \ - (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \ - } \ - \ - DO_FREE_HASH_NODE(_n); \ +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) + +#define FREE_HASH_NODE(_n) \ + do { \ + tfree(_n); \ } while (0); -static FORCE_INLINE void __wr_lock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { +struct SHashNode { + SHashNode *next; + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + uint16_t refCount; // reference count + int8_t removed; // flag to indicate removed + char data[]; +}; + +typedef struct SHashEntry { + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode *next; +} SHashEntry; + +struct SHashObj { + SHashEntry ** hashList; + size_t capacity; // number of slots + size_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + _hash_free_fn_t freeFp; // hash node free callback function + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type + bool enableUpdate; // enable update + SArray * pMemBlock; // memory block allocated for SHashEntry + _hash_before_fn_t callbackFp; // function invoked before return the value to caller +}; + +/* + * Function definition + */ +static FORCE_INLINE void taosHashWLock(SHashObj *pHashObj) { + if (pHashObj->type == HASH_NO_LOCK) { return; } - taosWLockLatch(lock); + taosWLockLatch(&pHashObj->lock); } -static FORCE_INLINE void __rd_lock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { +static FORCE_INLINE void taosHashWUnlock(SHashObj *pHashObj) { + if (pHashObj->type == HASH_NO_LOCK) { return; } - taosRLockLatch(lock); + + taosWUnLockLatch(&pHashObj->lock); } -static FORCE_INLINE void __rd_unlock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { +static FORCE_INLINE void taosHashRLock(SHashObj *pHashObj) { + if (pHashObj->type == HASH_NO_LOCK) { return; } - taosRUnLockLatch(lock); + + taosRLockLatch(&pHashObj->lock); } -static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) { - if (type == HASH_NO_LOCK) { +static FORCE_INLINE void taosHashRUnlock(SHashObj *pHashObj) { + if (pHashObj->type == HASH_NO_LOCK) { return; } - taosWUnLockLatch(lock); + + taosRUnLockLatch(&pHashObj->lock); +} + +static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry* pe) { + if (pHashObj->type == HASH_NO_LOCK) { + return; + } + taosWLockLatch(&pe->latch); +} + +static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry* pe) { + if (pHashObj->type == HASH_NO_LOCK) { + return; + } + + taosWUnLockLatch(&pe->latch); +} + +static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry* pe) { + if (pHashObj->type == HASH_NO_LOCK) { + return; + } + + taosRLockLatch(&pe->latch); +} + +static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry* pe) { + if (pHashObj->type == HASH_NO_LOCK) { + return; + } + + taosRUnLockLatch(&pe->latch); } static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = TMIN(length, HASH_MAX_CAPACITY); + int32_t len = MIN(length, HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); return i; } -static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, - uint32_t hashVal) { +static FORCE_INLINE SHashNode * +doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + if ((pNode->keyLen == keyLen) && + ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; @@ -90,60 +157,57 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntr } /** - * Resize the hash list if the threshold is reached + * resize the hash list if the threshold is reached * * @param pHashObj */ static void taosHashTableResize(SHashObj *pHashObj); /** + * allocate and initialize a hash node + * * @param key key of object for hash, usually a null-terminated string * @param keyLen length of key - * @param pData actually data. Requires a consecutive memory block, no pointer is allowed in pData. - * Pointer copy causes memory access error. + * @param pData data to be stored in hash node * @param dsize size of data * @return SHashNode */ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal); /** - * Update the hash node + * update the hash node * - * @param pNode hash node - * @param key key for generate hash value - * @param keyLen key length - * @param pData actual data - * @param dsize size of actual data - * @return hash node + * @param pHashObj hash table object + * @param pe hash table entry to operate on + * @param prev previous node + * @param pNode the old node with requested key + * @param pNewNode the new node with requested key */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode, - SHashNode *pNewNode) { +static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); - pNode->count--; + atomic_sub_fetch_32(&pNode->refCount, 1); if (prev != NULL) { prev->next = pNewNode; } else { pe->next = pNewNode; } - if (pNode->count <= 0) { + if (pNode->refCount <= 0) { pNewNode->next = pNode->next; - DO_FREE_HASH_NODE(pNode); + FREE_HASH_NODE(pNode); } else { pNewNode->next = pNode; pe->num++; - atomic_add_fetch_32(&pHashObj->size, 1); + atomic_add_fetch_64(&pHashObj->size, 1); } - - return pNewNode; } /** * insert the hash node at the front of the linked list * - * @param pHashObj - * @param pNode + * @param pHashObj hash table object + * @param pNode the old node with requested key */ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); @@ -156,47 +220,69 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj); /** - * Get the next element in hash table for iterator - * @param pIter + * + * @param pHashObj * @return */ +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { + return taosHashGetSize(pHashObj) == 0; +} SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { - assert(fn != NULL); + if (fn == NULL) { + assert(0); + return NULL; + } + if (capacity == 0) { capacity = 4; } SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj)); if (pHashObj == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } // the max slots is not defined by user pHashObj->capacity = taosHashCapacity((int32_t)capacity); - assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); + pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; + pHashObj->hashFp = fn; pHashObj->type = type; pHashObj->enableUpdate = update; + ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); + pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { free(pHashObj); - uError("failed to allocate memory, reason:%s", strerror(errno)); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; - } else { - pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); - - void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry)); - } - - taosArrayPush(pHashObj->pMemBlock, &p); } + pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); + if (pHashObj->pMemBlock == NULL) { + free(pHashObj->hashList); + free(pHashObj); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); + if (p == NULL) { + taosArrayDestroy(pHashObj->pMemBlock); + free(pHashObj->hashList); + free(pHashObj); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry)); + } + + taosArrayPush(pHashObj->pMemBlock, &p); return pHashObj; } @@ -206,16 +292,24 @@ void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) { } } -int32_t taosHashGetSize(const SHashObj *pHashObj) { - if (!pHashObj) { - return 0; +void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp) { + if (pHashObj != NULL && fp != NULL) { + pHashObj->freeFp = fp; } - return (int32_t)atomic_load_32(&pHashObj->size); } -static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; } +int32_t taosHashGetSize(const SHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + return (int32_t)atomic_load_64(&pHashObj->size); +} + +int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { + if (pHashObj == NULL || key == NULL || keyLen == 0) { + return -1; + } -int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) { uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); if (pNewNode == NULL) { @@ -224,30 +318,32 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void // need the resize process, write lock applied if (HASH_NEED_RESIZE(pHashObj)) { - __wr_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashWLock(pHashObj); taosHashTableResize(pHashObj); - __wr_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashWUnlock(pHashObj); } - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + // disable resize + taosHashRLock(pHashObj); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } + taosHashEntryWLock(pHashObj, pe); SHashNode *pNode = pe->next; +#if 0 if (pe->num > 0) { assert(pNode != NULL); } else { assert(pNode == NULL); } +#endif - SHashNode *prev = NULL; + SHashNode* prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + if ((pNode->keyLen == keyLen) && + (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; @@ -260,24 +356,13 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table pushfrontNodeInEntryList(pe, pNewNode); + assert(pe->next != NULL); - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); // enable resize - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); - atomic_add_fetch_32(&pHashObj->size, 1); - - if (newAdded) { - *newAdded = true; - } + taosHashRUnlock(pHashObj); + atomic_add_fetch_64(&pHashObj->size, 1); return 0; } else { @@ -285,131 +370,66 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void if (pHashObj->enableUpdate) { doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); } else { - DO_FREE_HASH_NODE(pNewNode); + FREE_HASH_NODE(pNewNode); } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); // enable resize - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); - - if (newAdded) { - *newAdded = false; - } - + taosHashRUnlock(pHashObj); return pHashObj->enableUpdate ? 0 : -2; } } -int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { - return taosHashPutImpl(pHashObj, key, keyLen, data, size, NULL); -} - -int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) { - return taosHashPutImpl(pHashObj, key, keyLen, data, size, newAdded); -} +static void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef); void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashGetClone(pHashObj, key, keyLen, NULL); + void* p = NULL; + return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, false); } -// TODO(yihaoDeng), merge with taosHashGetClone -void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, - size_t *sz) { - if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { +int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf) { + terrno = 0; + /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false); + return terrno; +} + +int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t* size) { + terrno = 0; + + /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false); + return terrno; +} + +void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // only add the read lock to disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; // no data, return directly if (atomic_load_32(&pe->num) == 0) { - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashRUnlock(pHashObj); return NULL; } char *data = NULL; + taosHashEntryRLock(pHashObj, pe); - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pe->latch); - } - - if (pe->num > 0) { - assert(pe->next != NULL); - } else { - assert(pe->next == NULL); - } - - SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); - if (pNode != NULL) { - if (fp != NULL) { - fp(GET_HASH_NODE_DATA(pNode)); - } - - if (*d == NULL) { - *sz = pNode->dataLen + EXT_SIZE; - *d = calloc(1, *sz); - } else if (*sz < pNode->dataLen) { - *sz = pNode->dataLen + EXT_SIZE; - *d = realloc(*d, *sz); - } - memcpy((char *)(*d), GET_HASH_NODE_DATA(pNode), pNode->dataLen); - // just make runtime happy - if ((*sz) - pNode->dataLen > 0) { - memset((char *)(*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen); - } - - data = GET_HASH_NODE_DATA(pNode); - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pe->latch); - } - - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); - return data; -} - -void *taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *d, bool acquire) { - if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { - return NULL; - } - - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); - - // only add the read lock to disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); - - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - SHashEntry *pe = pHashObj->hashList[slot]; - - // no data, return directly - if (atomic_load_32(&pe->num) == 0) { - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); - return NULL; - } - - char *data = NULL; - - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pe->latch); - } - +#if 0 if (pe->num > 0) { assert(pe->next != NULL); } else { assert(pe->next == NULL); } +#endif SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); if (pNode != NULL) { @@ -417,175 +437,102 @@ void *taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, v pHashObj->callbackFp(GET_HASH_NODE_DATA(pNode)); } - if (d != NULL) { - memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); + if (size != NULL) { + if (*d == NULL) { + *size = pNode->dataLen; + *d = calloc(1, *size); + if (*d == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } else if (*size < pNode->dataLen) { + *size = pNode->dataLen; + char* tmp = realloc(*d, *size); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + *d = tmp; + } } - if (acquire) { - atomic_add_fetch_16(&pNode->count, 1); + if (addRef) { + atomic_add_fetch_16(&pNode->refCount, 1); + } + + if (*d != NULL) { + memcpy(*d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); } data = GET_HASH_NODE_DATA(pNode); } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pe->latch); - } + taosHashEntryRUnlock(pHashObj, pe); + taosHashRUnlock(pHashObj); - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return data; } -void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *d) { - return taosHashGetCloneImpl(pHashObj, key, keyLen, d, false); -} - -void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashGetCloneImpl(pHashObj, key, keyLen, NULL, true); -} - -int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen /*, void *data, size_t dsize*/) { - if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL || keyLen == 0) { return -1; } uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } + taosHashEntryWLock(pHashObj, pe); // double check after locked if (pe->num == 0) { assert(pe->next == NULL); - taosWUnLockLatch(&pe->latch); - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashEntryWUnlock(pHashObj, pe); + taosHashRUnlock(pHashObj); return -1; } - int32_t code = -1; + int code = -1; SHashNode *pNode = pe->next; SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && - pNode->removed == 0) - break; + if ((pNode->keyLen == keyLen) && + ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + pNode->removed == 0) { + code = 0; // it is found - prevNode = pNode; - pNode = pNode->next; - } + atomic_sub_fetch_32(&pNode->refCount, 1); + pNode->removed = 1; + if (pNode->refCount <= 0) { + if (prevNode == NULL) { + pe->next = pNode->next; + } else { + prevNode->next = pNode->next; + } - if (pNode) { - code = 0; // it is found - - pNode->count--; - pNode->removed = 1; - if (pNode->count <= 0) { - if (prevNode) { - prevNode->next = pNode->next; - } else { - pe->next = pNode->next; + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pNode); } - - // if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); - - pe->num--; - atomic_sub_fetch_32(&pHashObj->size, 1); - FREE_HASH_NODE(pHashObj, pNode); + } else { + prevNode = pNode; + pNode = pNode->next; } } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashEntryWUnlock(pHashObj, pe); + taosHashRUnlock(pHashObj); return code; } -int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { - if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { - return 0; - } - - // disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); - - int32_t numOfEntries = (int32_t)pHashObj->capacity; - for (int32_t i = 0; i < numOfEntries; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - if (pEntry->num == 0) { - continue; - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pEntry->latch); - } - - // todo remove the first node - SHashNode *pNode = NULL; - while ((pNode = pEntry->next) != NULL) { - if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { - pEntry->num -= 1; - atomic_sub_fetch_32(&pHashObj->size, 1); - - pEntry->next = pNode->next; - - if (pEntry->num == 0) { - assert(pEntry->next == NULL); - } else { - assert(pEntry->next != NULL); - } - - FREE_HASH_NODE(pHashObj, pNode); - } else { - break; - } - } - - // handle the following node - if (pNode != NULL) { - assert(pNode == pEntry->next); - SHashNode *pNext = NULL; - - while ((pNext = pNode->next) != NULL) { - // not qualified, remove it - if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) { - pNode->next = pNext->next; - pEntry->num -= 1; - atomic_sub_fetch_32(&pHashObj->size, 1); - - if (pEntry->num == 0) { - assert(pEntry->next == NULL); - } else { - assert(pEntry->next != NULL); - } - - FREE_HASH_NODE(pHashObj, pNext); - } else { - pNode = pNext; - } - } - } - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pEntry->latch); - } - } - - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); - return 0; -} - void taosHashClear(SHashObj *pHashObj) { if (pHashObj == NULL) { return; @@ -593,12 +540,12 @@ void taosHashClear(SHashObj *pHashObj) { SHashNode *pNode, *pNext; - __wr_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashWLock(pHashObj); for (int32_t i = 0; i < pHashObj->capacity; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; if (pEntry->num == 0) { - assert(pEntry->next == 0); + assert(pEntry->next == NULL); continue; } @@ -607,7 +554,7 @@ void taosHashClear(SHashObj *pHashObj) { while (pNode) { pNext = pNode->next; - FREE_HASH_NODE(pHashObj, pNode); + FREE_HASH_NODE(pNode); pNode = pNext; } @@ -616,10 +563,11 @@ void taosHashClear(SHashObj *pHashObj) { pEntry->next = NULL; } - atomic_store_32(&pHashObj->size, 0); - __wr_unlock((void *)&pHashObj->lock, pHashObj->type); + pHashObj->size = 0; + taosHashWUnlock(pHashObj); } +// the input paras should be SHashObj **, so the origin input will be set by tfree(*pHashObj) void taosHashCleanup(SHashObj *pHashObj) { if (pHashObj == NULL) { return; @@ -636,26 +584,29 @@ void taosHashCleanup(SHashObj *pHashObj) { } taosArrayDestroy(pHashObj->pMemBlock); - - memset(pHashObj, 0, sizeof(SHashObj)); free(pHashObj); } // for profile only -int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { +int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){ if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return 0; } int32_t num = 0; + taosHashRLock((SHashObj*) pHashObj); for (int32_t i = 0; i < pHashObj->size; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; + + // fine grain per entry lock is not held since this is used + // for profiling only and doesn't need an accurate count. if (num < pEntry->num) { num = pEntry->num; } } + taosHashRUnlock((SHashObj*) pHashObj); return num; } @@ -664,28 +615,24 @@ void taosHashTableResize(SHashObj *pHashObj) { return; } - // double the original capacity - SHashNode *pNode = NULL; - SHashNode *pNext = NULL; - - int32_t newSize = (int32_t)(pHashObj->capacity << 1u); - if (newSize > HASH_MAX_CAPACITY) { - // uDebug("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", - // pHashObj->capacity, HASH_MAX_CAPACITY); + int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); + if (newCapacity > HASH_MAX_CAPACITY) { +// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", +// pHashObj->capacity, HASH_MAX_CAPACITY); return; } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); - if (pNewEntryList == NULL) { // todo handle error - // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity); + if (pNewEntryList == NULL) { +// uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } pHashObj->hashList = pNewEntryList; - size_t inc = newSize - pHashObj->capacity; - void *p = calloc(inc, sizeof(SHashEntry)); + size_t inc = newCapacity - pHashObj->capacity; + void * p = calloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -693,92 +640,62 @@ void taosHashTableResize(SHashObj *pHashObj) { taosArrayPush(pHashObj->pMemBlock, &p); - pHashObj->capacity = newSize; - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry *pe = pHashObj->hashList[i]; - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } + pHashObj->capacity = newCapacity; + for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { + SHashEntry *pe = pHashObj->hashList[idx]; + SHashNode *pNode; + SHashNode *pNext; + SHashNode *pPrev = NULL; if (pe->num == 0) { assert(pe->next == NULL); continue; } - while ((pNode = pe->next) != NULL) { - int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); - if (j != i) { - pe->num -= 1; - pe->next = pNode->next; + pNode = pe->next; - if (pe->num == 0) { - assert(pe->next == NULL); + assert(pNode != NULL); + + while (pNode != NULL) { + int32_t newIdx = HASH_INDEX(pNode->hashVal, pHashObj->capacity); + pNext = pNode->next; + if (newIdx != idx) { + pe->num -= 1; + if (pPrev == NULL) { + pe->next = pNext; } else { - assert(pe->next != NULL); + pPrev->next = pNext; } - SHashEntry *pNewEntry = pHashObj->hashList[j]; + SHashEntry *pNewEntry = pHashObj->hashList[newIdx]; pushfrontNodeInEntryList(pNewEntry, pNode); } else { - break; - } - } - - if (pNode != NULL) { - while ((pNext = pNode->next) != NULL) { - int32_t j = HASH_INDEX(pNext->hashVal, pHashObj->capacity); - if (j != i) { - pe->num -= 1; - - pNode->next = pNext->next; - pNext->next = NULL; - - // added into new slot - SHashEntry *pNewEntry = pHashObj->hashList[j]; - - if (pNewEntry->num == 0) { - assert(pNewEntry->next == NULL); - } else { - assert(pNewEntry->next != NULL); - } - - pushfrontNodeInEntryList(pNewEntry, pNext); - } else { - pNode = pNext; - } - } - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); + pPrev = pNode; } + pNode = pNext; } } int64_t et = taosGetTimestampUs(); - uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, - ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); +// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, +// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { SHashNode *pNewNode = malloc(sizeof(SHashNode) + keyLen + dsize); if (pNewNode == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pNewNode->keyLen = (uint32_t)keyLen; + pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; pNewNode->dataLen = (uint32_t)dsize; - pNewNode->count = 1; + pNewNode->refCount= 1; pNewNode->removed = 0; - pNewNode->next = NULL; + pNewNode->next = NULL; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); @@ -800,51 +717,32 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return 0; } - return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + - sizeof(SHashObj); + return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void*))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } -FORCE_INLINE int32_t taosHashGetKey(void *data, void **key, size_t *keyLen) { - if (NULL == data || NULL == key) { - return -1; - } - - SHashNode *node = GET_HASH_PNODE(data); - *key = GET_HASH_NODE_KEY(node); - if (keyLen) { +void *taosHashGetKey(void *data, size_t* keyLen) { + SHashNode * node = GET_HASH_PNODE(data); + if (keyLen != NULL) { *keyLen = node->keyLen; } - return 0; -} - -FORCE_INLINE int32_t taosHashGetDataLen(void *data) { - SHashNode *node = GET_HASH_PNODE(data); - return node->keyLen; -} - -FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { - SHashNode *node = GET_HASH_PNODE(data); - return node->keyLen; + return GET_HASH_NODE_KEY(node); } // release the pNode, return next pNode, and lock the current entry -static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { +static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p); SHashNode *prevNode = NULL; *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[*slot]; - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } + taosHashEntryWLock(pHashObj, pe); SHashNode *pNode = pe->next; - while (pNode) { - if (pNode == pOld) break; + if (pNode == pOld) + break; prevNode = pNode; pNode = pNode->next; @@ -857,8 +755,8 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { pNode = pNode->next; } - pOld->count--; - if (pOld->count <= 0) { + atomic_sub_fetch_32(&pOld->refCount, 1); + if (pOld->refCount <=0) { if (prevNode) { prevNode->next = pOld->next; } else { @@ -866,11 +764,11 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { } pe->num--; - atomic_sub_fetch_32(&pHashObj->size, 1); - FREE_HASH_NODE(pHashObj, pOld); + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pOld); } } else { - uError("pNode:%p data:%p is not there!!!", pNode, p); +// uError("pNode:%p data:%p is not there!!!", pNode, p); } return pNode; @@ -879,20 +777,18 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pHashObj == NULL) return NULL; - int32_t slot = 0; + int slot = 0; char *data = NULL; // only add the read lock to disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); SHashNode *pNode = NULL; if (p) { pNode = taosHashReleaseNode(pHashObj, p, &slot); if (pNode == NULL) { SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); slot = slot + 1; } @@ -902,10 +798,7 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { for (; slot < pHashObj->capacity; ++slot) { SHashEntry *pe = pHashObj->hashList[slot]; - // lock entry - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } + taosHashEntryWLock(pHashObj, pe); pNode = pe->next; while (pNode) { @@ -915,23 +808,22 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pNode) break; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); } } if (pNode) { SHashEntry *pe = pHashObj->hashList[slot]; - uint16_t prevRef = atomic_load_16(&pNode->count); - uint16_t afterRef = atomic_add_fetch_16(&pNode->count, 1); + uint16_t prevRef = atomic_load_16(&pNode->refCount); + uint16_t afterRef = atomic_add_fetch_16(&pNode->refCount, 1); + ASSERT(prevRef < afterRef); // the reference count value is overflow, which will cause the delete node operation immediately. if (prevRef > afterRef) { uError("hash entry ref count overflow, prev ref:%d, current ref:%d", prevRef, afterRef); // restore the value - atomic_sub_fetch_16(&pNode->count, 1); + atomic_sub_fetch_16(&pNode->refCount, 1); data = NULL; } else { data = GET_HASH_NODE_DATA(pNode); @@ -941,12 +833,10 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { uWarn("hash entry ref count is abnormally high: %d", afterRef); } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } + taosHashEntryWUnlock(pHashObj, pe); } - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashRUnlock(pHashObj); return data; } @@ -954,17 +844,21 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { if (pHashObj == NULL || p == NULL) return; // only add the read lock to disable the resize process - __rd_lock((void *)&pHashObj->lock, pHashObj->type); + taosHashRLock(pHashObj); - int32_t slot; + int slot; taosHashReleaseNode(pHashObj, p, &slot); SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - __rd_unlock((void *)&pHashObj->lock, pHashObj->type); + taosHashEntryWUnlock(pHashObj, pe); + taosHashRUnlock(pHashObj); +} + +//TODO remove it +void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { + void* p = NULL; + return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true); } void taosHashRelease(SHashObj *pHashObj, void *p) { taosHashCancelIterate(pHashObj, p); } diff --git a/source/util/src/thttp.c b/source/util/src/thttp.c index adf29b1aa9..0737f67ed1 100644 --- a/source/util/src/thttp.c +++ b/source/util/src/thttp.c @@ -63,7 +63,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, goto SEND_OVER; } - uInfo("send http to %s:%u, len:%d content: %s", server, port, contLen, pCont); + uTrace("send http to %s:%u, len:%d content: %s", server, port, contLen, pCont); code = 0; SEND_OVER: diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index d821e0440b..7f5664b9cd 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -17,12 +17,10 @@ #include "tlog.h" #include "tutil.h" -#define LOG_MAX_LINE_SIZE (1000) -#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 10) -#define LOG_MAX_LINE_CONTENT_SIZE (LOG_MAX_LINE_SIZE - 100) -#define LOG_MAX_LINE_DUMP_SIZE (65 * 1024) -#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 10) -#define LOG_MAX_LINE_DUMP_CONTENT_SIZE (LOG_MAX_LINE_DUMP_SIZE - 100) +#define LOG_MAX_LINE_SIZE (1024) +#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) +#define LOG_MAX_LINE_DUMP_SIZE (65 * 1024) +#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 3) #define LOG_FILE_NAME_LEN 300 #define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB @@ -66,13 +64,18 @@ typedef struct { static int8_t tsLogInited = 0; static SLogObj tsLogObj = {.fileNum = 1}; +static int64_t tsAsyncLogLostLines = 0; +static int32_t tsWriteInterval = LOG_DEFAULT_INTERVAL; -int8_t tscEmbeddedInUtil = 0; -int32_t tsLogKeepDays = 0; +bool tsLogEmbedded = 0; bool tsAsyncLog = true; int32_t tsNumOfLogLines = 10000000; -int64_t tsAsyncLogLostLines = 0; -int32_t tsWriteInterval = LOG_DEFAULT_INTERVAL; +int32_t tsLogKeepDays = 0; +LogFp tsLogFp = NULL; +int64_t tsNumOfErrorLogs = 0; +int64_t tsNumOfInfoLogs = 0; +int64_t tsNumOfDebugLogs = 0; +int64_t tsNumOfTraceLogs = 0; // log int32_t dDebugFlag = 135; @@ -97,7 +100,7 @@ int64_t dbgBigWN = 0; int64_t dbgWSize = 0; static void *taosAsyncOutputLog(void *param); -static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); +static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(TdFilePtr pFile); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); @@ -177,7 +180,7 @@ static void taosKeepOldLog(char *oldName) { char compressFileName[LOG_FILE_NAME_LEN + 20]; snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); if (taosCompressFile(fileName, compressFileName) == 0) { - (void)remove(fileName); + (void)taosRemoveFile(fileName); } } @@ -248,7 +251,7 @@ void taosResetLog() { tsLogObj.lines = tsLogObj.maxLines + 10; taosOpenNewLogFile(); - (void)remove(lastName); + (void)taosRemoveFile(lastName); uInfo("=================================="); uInfo(" reset log file "); @@ -372,44 +375,41 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { return 0; } -void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { - if (!osLogSpaceAvailable()) return; +static void taosUpdateLogNums(ELogLevel level) { + switch (level) { + case DEBUG_ERROR: + atomic_add_fetch_64(&tsNumOfErrorLogs, 1); + break; + case DEBUG_INFO: + atomic_add_fetch_64(&tsNumOfInfoLogs, 1); + break; + case DEBUG_DEBUG: + atomic_add_fetch_64(&tsNumOfDebugLogs, 1); + break; + case DEBUG_DUMP: + case DEBUG_TRACE: + atomic_add_fetch_64(&tsNumOfTraceLogs, 1); + break; + default: + break; + } +} - va_list argpointer; - char buffer[LOG_MAX_LINE_BUFFER_SIZE] = {0}; - int32_t len; +static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { struct tm Tm, *ptm; struct timeval timeSecs; - time_t curTime; taosGetTimeOfDay(&timeSecs); - curTime = timeSecs.tv_sec; + time_t curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); - len += sprintf(buffer + len, "%s", flags); - - va_start(argpointer, format); - int32_t writeLen = vsnprintf(buffer + len, LOG_MAX_LINE_CONTENT_SIZE, format, argpointer); - if (writeLen <= 0) { - char tmp[LOG_MAX_LINE_DUMP_BUFFER_SIZE] = {0}; - writeLen = vsnprintf(tmp, LOG_MAX_LINE_DUMP_CONTENT_SIZE, format, argpointer); - strncpy(buffer + len, tmp, LOG_MAX_LINE_CONTENT_SIZE); - len += LOG_MAX_LINE_CONTENT_SIZE; - } else if (writeLen >= LOG_MAX_LINE_CONTENT_SIZE) { - len += LOG_MAX_LINE_CONTENT_SIZE; - } else { - len += writeLen; - } - va_end(argpointer); - - if (len > LOG_MAX_LINE_SIZE) len = LOG_MAX_LINE_SIZE; - - buffer[len++] = '\n'; - buffer[len] = 0; + return sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId(), flags); +} +static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL) { + taosUpdateLogNums(level); if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { @@ -418,16 +418,63 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { if (tsLogObj.maxLines > 0) { atomic_add_fetch_32(&tsLogObj.lines, 1); - - if ((tsLogObj.lines > tsLogObj.maxLines) && (tsLogObj.openInProgress == 0)) taosOpenNewLogFile(); + if ((tsLogObj.lines > tsLogObj.maxLines) && (tsLogObj.openInProgress == 0)) { + taosOpenNewLogFile(); + } } } - if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) { + write(1, buffer, (uint32_t)len); + } +} + +void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) { + if (!osLogSpaceAvailable()) return; + if (!(dflag & DEBUG_FILE) && !(dflag & DEBUG_SCREEN)) return; + + char buffer[LOG_MAX_LINE_BUFFER_SIZE]; + int32_t len = taosBuildLogHead(buffer, flags); + + va_list argpointer; + va_start(argpointer, format); + int32_t writeLen = len + vsnprintf(buffer + len, LOG_MAX_LINE_BUFFER_SIZE - len, format, argpointer); + va_end(argpointer); + + if (writeLen > LOG_MAX_LINE_SIZE) writeLen = LOG_MAX_LINE_SIZE; + buffer[writeLen++] = '\n'; + buffer[writeLen] = 0; + + taosPrintLogImp(level, dflag, buffer, writeLen); + + if (tsLogFp && level <= DEBUG_INFO) { + buffer[writeLen - 1] = 0; + (*tsLogFp)(taosGetTimestampMs(), level, buffer + len); + } +} + +void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) { + if (!osLogSpaceAvailable()) return; + if (!(dflag & DEBUG_FILE) && !(dflag & DEBUG_SCREEN)) return; + + char buffer[LOG_MAX_LINE_DUMP_BUFFER_SIZE]; + int32_t len = taosBuildLogHead(buffer, flags); + + va_list argpointer; + va_start(argpointer, format); + len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_BUFFER_SIZE - len, format, argpointer); + va_end(argpointer); + + if (len > LOG_MAX_LINE_DUMP_SIZE) len = LOG_MAX_LINE_DUMP_SIZE; + buffer[len++] = '\n'; + buffer[len] = 0; + + taosPrintLogImp(level, dflag, buffer, len); } void taosDumpData(unsigned char *msg, int32_t len) { if (!osLogSpaceAvailable()) return; + taosUpdateLogNums(DEBUG_DUMP); char temp[256]; int32_t i, pos = 0, c = 0; @@ -449,50 +496,6 @@ void taosDumpData(unsigned char *msg, int32_t len) { taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); } -void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { - if (!osLogSpaceAvailable()) return; - - va_list argpointer; - char buffer[LOG_MAX_LINE_DUMP_BUFFER_SIZE]; - int32_t len; - struct tm Tm, *ptm; - struct timeval timeSecs; - time_t curTime; - - taosGetTimeOfDay(&timeSecs); - curTime = timeSecs.tv_sec; - ptm = localtime_r(&curTime, &Tm); - - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); - len += sprintf(buffer + len, "%s", flags); - - va_start(argpointer, format); - len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_CONTENT_SIZE, format, argpointer); - va_end(argpointer); - - if (len > LOG_MAX_LINE_DUMP_SIZE) len = LOG_MAX_LINE_DUMP_SIZE; - - buffer[len++] = '\n'; - buffer[len] = 0; - - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL) { - if (tsAsyncLog) { - taosPushLogBuffer(tsLogObj.logHandle, buffer, len); - } else { - taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); - } - - if (tsLogObj.maxLines > 0) { - atomic_add_fetch_32(&tsLogObj.lines, 1); - - if ((tsLogObj.lines > tsLogObj.maxLines) && (tsLogObj.openInProgress == 0)) taosOpenNewLogFile(); - } - } - - if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); -} - static void taosCloseLogByFd(TdFilePtr pFile) { if (pFile != NULL) { taosUnLockLogFile(pFile); @@ -525,7 +528,7 @@ _err: return NULL; } -static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, char *msg, int32_t msgLen) { +static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, const char *msg, int32_t msgLen) { if (start > end) { memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); } else { @@ -539,7 +542,7 @@ static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, ch LOG_BUF_END(tLogBuff) = (LOG_BUF_END(tLogBuff) + msgLen) % LOG_BUF_SIZE(tLogBuff); } -static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) { +static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, const char *msg, int32_t msgLen) { int32_t start = 0; int32_t end = 0; int32_t remainSize = 0; diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 45c38165a1..8c2fd7809d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -42,8 +42,8 @@ struct SDiskbasedBuf { bool comp; // compressed before flushed to disk uint64_t nextPos; // next page flush position - uint64_t qId; // for debug purpose - bool printStatis; // Print statistics info when closing this buffer. + char* id; // for debug purpose + bool printStatis; // Print statistics info when closing this buffer. SDiskbasedBufStatis statis; }; @@ -269,11 +269,12 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pag SPageInfo* ppi = malloc(sizeof(SPageInfo)); ppi->pageId = pageId; - ppi->pData = NULL; + ppi->pData = NULL; ppi->offset = -1; ppi->length = -1; - ppi->used = true; - ppi->pn = NULL; + ppi->used = true; + ppi->pn = NULL; + ppi->dirty = false; return *(SPageInfo**)taosArrayPush(list, &ppi); } @@ -356,7 +357,7 @@ static SPageInfo* getPageInfoFromPayload(void* page) { return ppi; } -int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir) { *pBuf = calloc(1, sizeof(SDiskbasedBuf)); @@ -366,13 +367,13 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem } pPBuf->pageSize = pagesize; - pPBuf->numOfPages = 0; // all pages are in buffer in the first place + pPBuf->numOfPages = 0; // all pages are in buffer in the first place pPBuf->totalBufSize = 0; pPBuf->inMemPages = inMemBufSize / pagesize; // maximum allowed pages, it is a soft limit. pPBuf->allocateId = -1; - pPBuf->comp = true; - pPBuf->pFile = NULL; - pPBuf->qId = qId; + pPBuf->comp = true; + pPBuf->pFile = NULL; + pPBuf->id = strdup(id); pPBuf->fileSize = 0; pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem)); pPBuf->freePgList = tdListNew(POINTER_BYTES); @@ -471,7 +472,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { return (void*)(GET_DATA_PAYLOAD(*pi)); } else { // not in memory - assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->length >= 0 && (*pi)->offset >= 0); + assert((*pi)->pData == NULL && (*pi)->pn == NULL && (((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1))); char* availablePage = NULL; if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) { @@ -493,9 +494,12 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { lruListPushFront(pBuf->lruList, *pi); (*pi)->used = true; - int32_t code = loadPageFromDisk(pBuf, *pi); - if (code != 0) { - return NULL; + // some data has been flushed to disk, and needs to be loaded into buffer again. + if ((*pi)->length > 0 && (*pi)->offset >= 0) { + int32_t code = loadPageFromDisk(pBuf, *pi); + if (code != 0) { + return NULL; + } } return (void*)(GET_DATA_PAYLOAD(*pi)); @@ -540,13 +544,13 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { if (pBuf->pFile != NULL) { uDebug( "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page " - "size:%.2f Kb, %" PRIx64 "\n", + "size:%.2f Kb, %s\n", pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, - listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId); + listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->id); taosCloseFile(&pBuf->pFile); } else { - uDebug("Paged buffer closed, total:%.2f Kb, no file created, %" PRIx64, pBuf->totalBufSize / 1024.0, pBuf->qId); + uDebug("Paged buffer closed, total:%.2f Kb, no file created, %s", pBuf->totalBufSize / 1024.0, pBuf->id); } // print the statistics information @@ -559,7 +563,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { ps->loadPages, ps->loadBytes / (1024.0 * ps->loadPages)); } - remove(pBuf->path); + taosRemoveFile(pBuf->path); tfree(pBuf->path); SArray** p = taosHashIterate(pBuf->groupSet, NULL); @@ -584,6 +588,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { taosHashCleanup(pBuf->groupSet); taosHashCleanup(pBuf->all); + tfree(pBuf->id); tfree(pBuf->assistBuf); tfree(pBuf); } @@ -639,9 +644,9 @@ void dBufPrintStatis(const SDiskbasedBuf* pBuf) { printf( "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f " - "Kb, %" PRIx64 "\n", + "Kb, %s\n", pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, - listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId); + listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->id); printf( "Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n", diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index abb42ef28d..72c18518e3 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -19,41 +19,41 @@ #include "tlog.h" #include "tsched.h" -#define tmrFatal(...) \ - { \ - if (tmrDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrFatal(...) \ + { \ + if (tmrDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TMR FATAL ", DEBUG_FATAL, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrError(...) \ - { \ - if (tmrDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrError(...) \ + { \ + if (tmrDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TMR ERROR ", DEBUG_ERROR, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrWarn(...) \ - { \ - if (tmrDebugFlag & DEBUG_WARN) { \ - taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrWarn(...) \ + { \ + if (tmrDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TMR WARN ", DEBUG_WARN, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrInfo(...) \ - { \ - if (tmrDebugFlag & DEBUG_INFO) { \ - taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrInfo(...) \ + { \ + if (tmrDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TMR ", DEBUG_INFO, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrDebug(...) \ - { \ - if (tmrDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrDebug(...) \ + { \ + if (tmrDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TMR ", DEBUG_DEBUG, tmrDebugFlag, __VA_ARGS__); \ + } \ } -#define tmrTrace(...) \ - { \ - if (tmrDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \ - } \ +#define tmrTrace(...) \ + { \ + if (tmrDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TMR ", DEBUG_TRACE, tmrDebugFlag, __VA_ARGS__); \ + } \ } #define TIMER_STATE_WAITING 0 diff --git a/source/util/test/arrayTest.cpp b/source/util/test/arrayTest.cpp index 939d4a701d..d8ace09392 100644 --- a/source/util/test/arrayTest.cpp +++ b/source/util/test/arrayTest.cpp @@ -43,6 +43,9 @@ static void remove_batch_test() { taosArrayPush(delList, &a); taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList)); EXPECT_EQ(taosArrayGetSize(pa), 17); + + taosArrayDestroy(pa); + taosArrayDestroy(delList); } } // namespace @@ -79,4 +82,6 @@ TEST(arrayTest, array_search_test) { } } + + taosArrayDestroy(pa); } diff --git a/source/util/test/cacheTest.cpp b/source/util/test/cacheTest.cpp index 970f1c23a9..e1c8c8c14e 100644 --- a/source/util/test/cacheTest.cpp +++ b/source/util/test/cacheTest.cpp @@ -1,12 +1,12 @@ -#include "os.h" #include #include +#include "os.h" #include "taos.h" #include "tcache.h" // test cache -TEST(testCase, client_cache_test) { +TEST(cacheTest, client_cache_test) { const int32_t REFRESH_TIME_IN_SEC = 2; SCacheObj* tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test"); @@ -92,7 +92,7 @@ TEST(testCase, client_cache_test) { taosCacheCleanup(tscMetaCache); } -TEST(testCase, cache_resize_test) { +TEST(cacheTest, cache_iter_test) { const int32_t REFRESH_TIME_IN_SEC = 2; auto* pCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, false, NULL, "test"); @@ -107,6 +107,7 @@ TEST(testCase, cache_resize_test) { int32_t len = sprintf(key, "abc_%7d", i); taosCachePut(pCache, key, strlen(key), data, len, 3600); } + uint64_t endTime = taosGetTimestampUs(); printf("add %d object cost:%" PRIu64 " us, avg:%f us\n", num, endTime - startTime, (endTime-startTime)/(double)num); @@ -120,5 +121,22 @@ TEST(testCase, cache_resize_test) { endTime = taosGetTimestampUs(); printf("retrieve %d object cost:%" PRIu64 " us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num); + int32_t count = 0; + SCacheIter* pIter = taosCacheCreateIter(pCache); + while(taosCacheIterNext(pIter)) { + size_t keyLen = 0; + size_t dataLen = 0; + + char* key1 = static_cast(taosCacheIterGetKey(pIter, &keyLen)); + char* data1 = static_cast(taosCacheIterGetData(pIter, &dataLen)); + +// char d[256] = {0}; +// memcpy(d, data1, dataLen); +// char k[256] = {0}; +// memcpy(k, key1, keyLen); + } + + ASSERT_EQ(count, num); + taosCacheCleanup(pCache); -} \ No newline at end of file +} diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 5505a6207f..46c95556d4 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -1,6 +1,6 @@ #include -#include "gtest/gtest.h" +#include #include "tencode.h" @@ -201,8 +201,8 @@ TEST(td_encode_test, encode_decode_cstr) { } } - delete buf; - delete cstr; + delete[] buf; + delete[] cstr; } typedef struct { @@ -354,7 +354,7 @@ static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { tEndDecode(pCoder); return 0; } - +#if 0 TEST(td_encode_test, compound_struct_encode_test) { SCoder encoder, decoder; uint8_t * buf1; @@ -436,5 +436,5 @@ TEST(td_encode_test, compound_struct_encode_test) { GTEST_ASSERT_EQ(dreq21.v_b, req2.v_b); tCoderClear(&decoder); } - +#endif #pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index cf11d6b5bf..7fd3d693fb 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -1,4 +1,4 @@ -#include "gtest/gtest.h" +#include #include "tfreelist.h" diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index ac1bae2434..f4d2d9b47b 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -1,9 +1,9 @@ -#include "os.h" #include #include -#include #include +#include "os.h" +#include "taosdef.h" #include "thash.h" #include "taos.h" @@ -106,7 +106,7 @@ void noLockPerformanceTest() { ASSERT_EQ(taosHashGetSize(hashTable), 0); char key[128] = {0}; - int32_t num = 5000000; + int32_t num = 5000; int64_t st = taosGetTimestampUs(); @@ -186,10 +186,15 @@ void acquireRleaseTest() { printf("%s,expect:%s", pdata->p, str3); ASSERT_TRUE(strcmp(pdata->p, str3) == 0); - + + tfree(pdata->p); + taosHashRelease(hashTable, pdata); num = taosHashGetSize(hashTable); ASSERT_EQ(num, 1); + + taosHashCleanup(hashTable); + tfree(data.p); } } diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 4b9d8c5f51..f392aac7d1 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -12,145 +12,150 @@ namespace { // simple test void simpleTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4096, 1, "/tmp/"); + SDiskbasedBuf* pBuf = NULL; + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", "/tmp/"); int32_t pageId = 0; int32_t groupId = 0; - SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* pBufPage = static_cast(getNewBufPage(pBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); - ASSERT_EQ(getTotalBufSize(pResultBuf), 1024); + ASSERT_EQ(getTotalBufSize(pBuf), 1024); - SIDList list = getDataBufPagesIdList(pResultBuf, groupId); + SIDList list = getDataBufPagesIdList(pBuf, groupId); ASSERT_EQ(taosArrayGetSize(list), 1); - ASSERT_EQ(getNumOfBufGroupId(pResultBuf), 1); + ASSERT_EQ(getNumOfBufGroupId(pBuf), 1); - releaseBufPage(pResultBuf, pBufPage); + releaseBufPage(pBuf, pBufPage); - SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); - SFilePage* t = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* t = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t == pBufPage1); - SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t1 == pBufPage2); - SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t2 == pBufPage3); - SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t3 == pBufPage4); - SFilePage* pBufPage5 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); + releaseBufPage(pBuf, pBufPage2); + + SFilePage* pBufPage5 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t4 == pBufPage5); - destroyDiskbasedBuf(pResultBuf); + destroyDiskbasedBuf(pBuf); } void writeDownTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + SDiskbasedBuf* pBuf = NULL; + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; int32_t groupId = 0; int32_t nx = 12345; - SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* pBufPage = static_cast(getNewBufPage(pBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); *(int32_t*)(pBufPage->data) = nx; writePageId = pageId; - releaseBufPage(pResultBuf, pBufPage); + + setBufPageDirty(pBufPage, true); + releaseBufPage(pBuf, pBufPage); - SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); - SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); - SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); - SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); - releaseBufPage(pResultBuf, t4); + releaseBufPage(pBuf, t4); // flush the written page to disk, and read it out again - SFilePage* pBufPagex = static_cast(getBufPage(pResultBuf, writePageId)); + SFilePage* pBufPagex = static_cast(getBufPage(pBuf, writePageId)); ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); - SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf, groupId); ASSERT_EQ(taosArrayGetSize(pa), 5); - destroyDiskbasedBuf(pResultBuf); + destroyDiskbasedBuf(pBuf); } void recyclePageTest() { - SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + SDiskbasedBuf* pBuf = NULL; + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; int32_t groupId = 0; int32_t nx = 12345; - SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* pBufPage = static_cast(getNewBufPage(pBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); - releaseBufPage(pResultBuf, pBufPage); + releaseBufPage(pBuf, pBufPage); - SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); - SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); - SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); - SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); - releaseBufPage(pResultBuf, t4); + releaseBufPage(pBuf, t4); - SFilePage* pBufPage5 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t5 = static_cast(getBufPage(pResultBuf, pageId)); + SFilePage* pBufPage5 = static_cast(getNewBufPage(pBuf, groupId, &pageId)); + SFilePage* t5 = static_cast(getBufPage(pBuf, pageId)); ASSERT_TRUE(t5 == pBufPage5); ASSERT_TRUE(pageId == 5); + releaseBufPage(pBuf, t5); // flush the written page to disk, and read it out again - SFilePage* pBufPagex = static_cast(getBufPage(pResultBuf, writePageId)); + SFilePage* pBufPagex = static_cast(getBufPage(pBuf, writePageId)); *(int32_t*)(pBufPagex->data) = nx; writePageId = pageId; // update the data - releaseBufPage(pResultBuf, pBufPagex); + releaseBufPage(pBuf, pBufPagex); - SFilePage* pBufPagex1 = static_cast(getBufPage(pResultBuf, 1)); + SFilePage* pBufPagex1 = static_cast(getBufPage(pBuf, 1)); - SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf, groupId); ASSERT_EQ(taosArrayGetSize(pa), 6); - destroyDiskbasedBuf(pResultBuf); + destroyDiskbasedBuf(pBuf); } } // namespace diff --git a/tests b/tests index 08ed39f0a5..904e6f0e15 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 08ed39f0a5fcbbfb5a630b945ab3d1998d4b4136 +Subproject commit 904e6f0e152e8fe61edfe0a0a9ae497cfde2a72c diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index c0ab7ef461..36489a448b 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -93,8 +93,7 @@ static void shellCheckTablesSQLFile(const char *directoryName) { sprintf(shellTablesSQLFile, "%s/tables.sql", directoryName); - struct stat fstat; - if (stat(shellTablesSQLFile, &fstat) < 0) { + if (taosFStatFile(shellTablesSQLFile, NULL, NULL) < 0) { shellTablesSQLFile[0] = 0; } } @@ -109,13 +108,12 @@ static void shellMallocSQLFiles() static void shellGetDirectoryFileList(char *inputDir) { - struct stat fileStat; - if (stat(inputDir, &fileStat) < 0) { + if (!taosDirExist(inputDir)) { fprintf(stderr, "ERROR: %s not exist\n", inputDir); exit(0); } - if (fileStat.st_mode & S_IFDIR) { + if (taosIsDir(inputDir)) { shellCheckTablesSQLFile(inputDir); shellSQLFileNum = shellGetFilesNum(inputDir, "sql"); int totalSQLFileNum = shellSQLFileNum;