xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #8182 from taosdata/3.0_refact 

3.0 refact
This commit is contained in:
Hongze Cheng 2021-10-12 10:33:21 +08:00 committed by GitHub
parent bf80ed2026 bab6e54b13
commit 30f0a07a21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 1105 additions and 541 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CMakeLists.txt
View File

@ -35,6 +35,18 @@ cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${DEPS_TMP_FILE})
## cJson ## cJson
cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE}) cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE})
## leveldb
option(BUILD_WITH_LEVELDB "If build with leveldb" OFF)
if(${BUILD_WITH_LEVELDB})
cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
endif(${BUILD_WITH_LEVELDB})
## rocksdb
option(BUILD_WITH_ROCKSDB "If build with rocksdb" ON)
if(${BUILD_WITH_ROCKSDB})
cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
endif(${BUILD_WITH_ROCKSDB})
## download dependencies ## download dependencies
configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt") configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .

1
cmake/EnableCMP0048.txt.in Normal file
View File

@ -0,0 +1 @@
cmake_policy(SET CMP0048 NEW)

13
cmake/leveldb_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,13 @@
# leveldb
ExternalProject_Add(leveldb
GIT_REPOSITORY https://github.com/taosdata-contrib/leveldb.git
GIT_TAG master
SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/leveldb"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

11
cmake/rocksdb_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,11 @@
# rocksdb
ExternalProject_Add(rocksdb
GIT_REPOSITORY https://github.com/taosdata-contrib/rocksdb.git
GIT_TAG v6.23.3
SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/rocksdb"
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

34
deps/CMakeLists.txt vendored
View File

@ -1,23 +1,57 @@
# googletest
if(${BUILD_TEST}) if(${BUILD_TEST})
add_subdirectory(googletest) add_subdirectory(googletest)
endif(${BUILD_TEST}) endif(${BUILD_TEST})
# cJson
# see https://stackoverflow.com/questions/37582508/silence-cmp0048-warnings-in-vendored-projects
set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
add_subdirectory(cJson) add_subdirectory(cJson)
target_include_directories( target_include_directories(
cjson cjson
# see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cJson> PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cJson>
) )
unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# lz4
add_subdirectory(lz4/build/cmake) add_subdirectory(lz4/build/cmake)
target_include_directories( target_include_directories(
lz4_static lz4_static
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
) )
# zlib
set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
add_subdirectory(zlib) add_subdirectory(zlib)
target_include_directories( target_include_directories(
zlib zlib
PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib
) )
unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# leveldb
if(${BUILD_WITH_LEVELDB})
option(LEVELDB_BUILD_TESTS "" OFF)
add_subdirectory(leveldb)
target_include_directories(
leveldb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
)
endif(${BUILD_WITH_LEVELDB})
# rocksdb
if(${BUILD_WITH_ROCKSDB})
option(WITH_TESTS "" OFF)
option(WITH_BENCHMARK_TOOLS "" OFF)
option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
add_subdirectory(rocksdb)
target_include_directories(
rocksdb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
)
endif(${BUILD_WITH_ROCKSDB})

36
include/common/tcompare.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TCOMPARE_H_
#define _TD_TCOMPARE_H_
#include "compare.h"
#include "ttypes.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);
__compar_fn_t getComparFunc(int32_t type, int32_t optr);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TCOMPARE_H_*/

45
include/common/trow.h
View File

@ -16,10 +16,55 @@
#ifndef _TD_COMMON_ROW_H_ #ifndef _TD_COMMON_ROW_H_
#define _TD_COMMON_ROW_H_ #define _TD_COMMON_ROW_H_
#include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
// types
typedef void * SRow;
typedef struct SRowBatch SRowBatch;
typedef struct SRowBuilder SRowBuilder;
typedef struct SRowBatchIter SRowBatchIter;
typedef struct SRowBatchBuilder SRowBatchBuilder;
// SRow
#define ROW_HEADER_SIZE (sizeof(uint8_t) + 2 * sizeof(uint16_t) + sizeof(uint64_t))
#define rowType(r) (*(uint8_t *)(r)) // row type
#define rowLen(r) (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t))) // row length
#define rowSVer(r) \
(*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t) + sizeof(uint16_t))) // row schema version, only for SDataRow
#define rowNCols(r) rowSVer(r) // only for SKVRow
#define rowVer(r) (*(uint64_t)POINTER_SHIFT(r, sizeof(uint8_t) + 2 * sizeof(uint16_t))) // row version
#define rowCopy(dest, r) memcpy((dest), r, rowLen(r))
static FORCE_INLINE SRow rowDup(SRow row) {
SRow r = malloc(rowLen(row));
if (r == NULL) {
return NULL;
}
rowCopy(r, row);
return r;
}
// SRowBatch
// SRowBuilder
SRowBuilder *rowBuilderCreate();
void rowBuilderDestroy(SRowBuilder *);
// SRowBatchIter
SRowBatchIter *rowBatchIterCreate(SRowBatch *);
void rowBatchIterDestroy(SRowBatchIter *);
const SRow rowBatchIterNext(SRowBatchIter *);
// SRowBatchBuilder
SRowBatchBuilder *rowBatchBuilderCreate();
void rowBatchBuilderDestroy(SRowBatchBuilder *);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

6
include/common/ttypes.h
View File

@ -6,10 +6,10 @@ extern "C" {
#endif #endif
#include "taosdef.h" #include "taosdef.h"
#include "types.h"
// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
typedef int32_t VarDataOffsetT; typedef int32_t VarDataOffsetT;
typedef int16_t VarDataLenT; // maxVarDataLen: 32767
typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535 typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535
typedef uint32_t TDRowTLenT; // total length, including overhead typedef uint32_t TDRowTLenT; // total length, including overhead
@ -30,11 +30,7 @@ typedef struct {
} SNCharNullT; } SNCharNullT;
#pragma pack(pop) #pragma pack(pop)
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
#define varDataLen(v) ((VarDataLenT *)(v))[0]
#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) #define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) #define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))

59
include/libs/tkv/tkv.h
View File

@ -20,52 +20,25 @@
extern "C" { extern "C" {
#endif #endif
typedef struct tkv_db_s tkv_db_t; // Types exported
typedef struct STkvDb STkvDb;
typedef struct STkvOptions STkvOptions;
typedef struct STkvCache STkvCache;
typedef struct { // DB operations
/* data */ STkvDb *tkvOpen(const STkvOptions *options, const char *path);
} tkv_key_t; void tkvClose(STkvDb *db);
void tkvPut(STkvDb *db, void * /*TODO*/);
typedef struct { // DB options
bool pinned; STkvOptions *tkvOptionsCreate();
int64_t ref; // TODO: use util library void tkvOptionsDestroy(STkvOptions *);
// TODO: add a RW latch here void tkvOptionsSetCache(STkvOptions *, STkvCache *);
uint64_t offset;
void * pObj;
} tkv_obj_t;
typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *); // DB cache
typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType;
typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type);
typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); void tkvCacheDestroy(STkvCache *);
typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *);
typedef void (*tkv_obj_destroy_fn_t)(void *);
typedef struct {
uint64_t memLimit;
tkv_get_key_fn_t getKey;
tkv_obj_encode_fn_t encode;
tkv_obj_decode_fn_t decode;
tkv_obj_comp_fn_t compare;
tkv_obj_destroy_fn_t destroy;
} tkv_db_option_t;
tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *);
int tkvCloseDB(tkv_db_t *);
int tkvPut(tkv_db_t *, tkv_obj_t *);
int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); // TODO: use array here
const tkv_obj_t *tkvGet(tkv_key_t *);
int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); // TODO: use array here
int tkvDrop(tkv_db_t *, tkv_key_t *);
int tkvDropBatch(tkv_db_t *, tkv_key_t **, int); // TODO: use array here
int tkvCommit(tkv_db_t *, void * /*TODO*/);
typedef struct {
} tkv_db_iter_t;
tkv_db_iter_t * tkvIterNew(tkv_db_t *);
void tkvIterFree(tkv_db_iter_t *);
const tkv_obj_t *tkvIterNext(tkv_db_iter_t *);
#ifdef __cplusplus #ifdef __cplusplus
} }

36
include/server/vnode/meta/meta.h
View File

@ -18,16 +18,42 @@
#include "taosmsg.h" #include "taosmsg.h"
#include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct SMeta SMeta; typedef uint64_t tuid_t;
int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq); // Types exported
int metaDropTable(SMeta *pMeta, SDropTableReq *pReq); typedef struct SMeta SMeta;
int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq); typedef struct SMetaOptions SMetaOptions;
int metaCommit(SMeta *pMeta); typedef struct SMetaQueryHandle SMetaQueryHandle;
typedef struct SMetaQueryOptions SMetaQueryOptions;
// SMeta operations
int metaCreate(const char *path);
int metaDestroy(const char *path);
SMeta *metaOpen(SMetaOptions *);
void metaClose(SMeta *);
int metaCreateTable(SMeta *, void *);
int metaDropTable(SMeta *, uint64_t tuid_t);
int metaAlterTable(SMeta *, void *);
int metaCommit(SMeta *);
// Options
SMetaOptions *metaOptionsCreate();
void metaOptionsDestroy(SMetaOptions *);
void metaOptionsSetCache(SMetaOptions *, size_t capacity);
// SMetaQueryHandle
SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOptions *);
void metaQueryHandleDestroy(SMetaQueryHandle *);
// SMetaQueryOptions
SMetaQueryOptions *metaQueryOptionsCreate();
void metaQueryOptionsDestroy(SMetaQueryOptions *);
#ifdef __cplusplus #ifdef __cplusplus
} }

75
include/server/vnode/tsdb/tsdb.h
View File

@ -23,33 +23,60 @@
extern "C" { extern "C" {
#endif #endif
typedef struct STsdb STsdb; // Types exported
typedef struct { typedef struct STsdb STsdb;
int32_t id; // TODO: use a global definition typedef struct STsdbOptions STsdbOptions;
int32_t days; typedef struct STsdbSMAOptions STsdbSMAOptions; // SMA stands for Small Materialized Aggregation
int32_t keep; typedef struct STsdbReadOptions STsdbReadOptions;
int32_t keep1; typedef struct STsdbSnapshot STsdbSnapshot;
int32_t keep2; typedef struct STsdbQueryHandle STsdbQueryHandle;
int32_t minRows;
int32_t maxRows;
int8_t precision;
int8_t update;
} STsdbCfg;
// Module init and clear // DB operations
int tsdbInit(); int tsdbCreate(const char *path);
int tsdbClear(); int tsdbDestroy(const char *path);
STsdb *tsdbOpen(const STsdbOptions *options);
void tsdbClose(STsdb *);
int tsdbReset(STsdb *, const STsdbOptions *);
int tsdbInsert(STsdb *, SSubmitReq *, SSubmitRsp *);
int tsdbCommit(STsdb *);
int tsdbCompact(STsdb *);
// Repository operations // Options
int tsdbCreateRepo(int id); STsdbOptions *tsdbOptionsCreate();
int tsdbDropRepo(int id); void tsdbOptionsDestroy(STsdbOptions *);
STsdb *tsdbOpenRepo(STsdbCfg *pCfg); void tsdbOptionsSetId(STsdbOptions *, int id);
int tsdbCloseRepo(STsdb *pTsdb); void tsdbOptionsSetHoursPerFile(STsdbOptions *, int hours);
int tsdbForceCloseRepo(STsdb *pTsdb); void tsdbOptionsSetRetention(STsdbOptions *, int keep, int keep1, int keep2);
void tsdbOptionsSetMinAndMaxRows(STsdbOptions *, int minRows, int maxRows);
void tsdbOptionsSetPrecision(STsdbOptions *, int);
void tsdbOptionsSetCache(STsdbOptions *, int);
typedef enum { TSDB_NO_UPDATE = 0, TSDB_WHOLE_ROW_UPDATE = 1, TSDB_PARTIAL_ROW_UPDATE = 2 } ETsdbUpdateType;
void tsdbOptionsSetUpdate(STsdbOptions *, ETsdbUpdateType);
void tsdbOptionsSetSMA(STsdbOptions *, STsdbSMAOptions *);
// Data commit // STsdbSMAOptions
int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg); STsdbSMAOptions *tsdbSMAOptionsCreate();
int tsdbCommit(STsdb *pTsdb); void tsdbSMAOptionsDestroy(STsdbSMAOptions *);
// void tsdbSMAOptionsSetFuncs(STsdbSMAOptions *, SArray * /*Array of function to perform on each block*/);
// void tsdbSMAOptionsSetIntervals(STsdbSMAOptions *, SArray *);
// void tsdbSMAOptionsSetColTypes(STsdbSMAOptions *, SArray *);
// STsdbQueryHandle
STsdbQueryHandle *tsdbQueryHandleCreate(STsdb *, STsdbReadOptions *);
void tsdbQueryHandleDestroy(STsdbQueryHandle *);
void tsdbResetQueryHandle(STsdbQueryHandle *, STsdbReadOptions *);
bool tsdbNextDataBlock(STsdbQueryHandle *);
// void tsdbGetDataBlockInfo(STsdbQueryHandle *, SDataBlockInfo *);
// void tsdbGetDataBlockStatisInfo(STsdbQueryHandle *, SDataStatis **);
// STsdbReadOptions
STsdbReadOptions *tsdbReadOptionsCreate();
void tsdbReadOptionsDestroy(STsdbReadOptions *);
void tsdbReadOptionsSetSnapshot(STsdbReadOptions *, STsdbSnapshot *);
// STsdbSnapshot
STsdbSnapshot *tsdbSnapshotCreate(STsdb *);
void tsdbSnapshotDestroy(STsdbSnapshot *);
#ifdef __cplusplus #ifdef __cplusplus
} }

19
include/util/tcompare.h → include/util/compare.h
View File

@ -46,11 +46,6 @@ int patternMatch(const char *pattern, const char *str, size_t size, const SPatte
int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
__compar_fn_t getComparFunc(int32_t type, int32_t optr);
int32_t taosArrayCompareString(const void* a, const void* b); int32_t taosArrayCompareString(const void* a, const void* b);
@ -82,12 +77,22 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
int32_t compareStrRegexComp(const void* pLeft, const void* pRight); int32_t compareStrRegexComp(const void* pLeft, const void* pRight);
int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight); int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight);
int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight); int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight);
int32_t compareFindItemInSet(const void *pLeft, const void* pRight); int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
int32_t compareInt16ValDesc(const void* pLeft, const void* pRight);
int32_t compareInt32ValDesc(const void* pLeft, const void* pRight);
int32_t compareInt64ValDesc(const void* pLeft, const void* pRight);
int32_t compareFloatValDesc(const void* pLeft, const void* pRight);
int32_t compareDoubleValDesc(const void* pLeft, const void* pRight);
int32_t compareUint8ValDesc(const void* pLeft, const void* pRight);
int32_t compareUint16ValDesc(const void* pLeft, const void* pRight);
int32_t compareUint32ValDesc(const void* pLeft, const void* pRight);
int32_t compareUint64ValDesc(const void* pLeft, const void* pRight);
int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight);
int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight);
#ifdef __cplusplus #ifdef __cplusplus
} }

27
include/util/tdef.h
View File

@ -110,33 +110,6 @@ do { \
(src) = (void *)((char *)src + sizeof(type));\ (src) = (void *)((char *)src + sizeof(type));\
} while(0) } while(0)
#define GET_INT8_VAL(x) (*(int8_t *)(x))
#define GET_INT16_VAL(x) (*(int16_t *)(x))
#define GET_INT32_VAL(x) (*(int32_t *)(x))
#define GET_INT64_VAL(x) (*(int64_t *)(x))
#define GET_UINT8_VAL(x) (*(uint8_t*) (x))
#define GET_UINT16_VAL(x) (*(uint16_t *)(x))
#define GET_UINT32_VAL(x) (*(uint32_t *)(x))
#define GET_UINT64_VAL(x) (*(uint64_t *)(x))
#ifdef _TD_ARM_32
float taos_align_get_float(const char* pBuf);
double taos_align_get_double(const char* pBuf);
#define GET_FLOAT_VAL(x) taos_align_get_float(x)
#define GET_DOUBLE_VAL(x) taos_align_get_double(x)
#define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); }
#define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
#define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); }
#define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
#else
#define GET_FLOAT_VAL(x) (*(float *)(x))
#define GET_DOUBLE_VAL(x) (*(double *)(x))
#define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); }
#define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); }
#define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); }
#define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
#endif
// TODO: check if below is necessary // TODO: check if below is necessary
#define TSDB_RELATION_INVALID 0 #define TSDB_RELATION_INVALID 0

88
include/util/types.h Normal file
View File

@ -0,0 +1,88 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TYPES_H_
#define _TD_TYPES_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
#define GET_INT8_VAL(x) (*(int8_t *)(x))
#define GET_INT16_VAL(x) (*(int16_t *)(x))
#define GET_INT32_VAL(x) (*(int32_t *)(x))
#define GET_INT64_VAL(x) (*(int64_t *)(x))
#define GET_UINT8_VAL(x) (*(uint8_t*) (x))
#define GET_UINT16_VAL(x) (*(uint16_t *)(x))
#define GET_UINT32_VAL(x) (*(uint32_t *)(x))
#define GET_UINT64_VAL(x) (*(uint64_t *)(x))
static FORCE_INLINE float taos_align_get_float(const char *pBuf) {
#if __STDC_VERSION__ >= 201112L
static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
#else
assert(sizeof(float) == sizeof(uint32_t));
#endif
float fv = 0;
memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem
return fv;
}
static FORCE_INLINE double taos_align_get_double(const char *pBuf) {
#if __STDC_VERSION__ >= 201112L
static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
#else
assert(sizeof(double) == sizeof(uint64_t));
#endif
double dv = 0;
memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem
return dv;
}
// #ifdef _TD_ARM_32
// float taos_align_get_float(const char* pBuf);
// double taos_align_get_double(const char* pBuf);
// #define GET_FLOAT_VAL(x) taos_align_get_float(x)
// #define GET_DOUBLE_VAL(x) taos_align_get_double(x)
// #define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); }
// #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
// #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); }
// #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
// #else
#define GET_FLOAT_VAL(x) (*(float *)(x))
#define GET_DOUBLE_VAL(x) (*(double *)(x))
#define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); }
#define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); }
#define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); }
#define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
// #endif
typedef int16_t VarDataLenT; // maxVarDataLen: 32767
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
#define varDataLen(v) ((VarDataLenT *)(v))[0]
#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
typedef int32_t VarDataOffsetT;
typedef int16_t VarDataLenT; // maxVarDataLen: 32767
#ifdef __cplusplus
}
#endif
#endif /*_TD_TYPES_H_*/

384
source/common/src/tcompare.c
View File

@ -12,334 +12,8 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#define _BSD_SOURCE
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "os.h"
#include "tcompare.h" #include "tcompare.h"
#include "ulog.h"
#include "thash.h"
#include "regex.h"
#include "ttypes.h"
int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
}
int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
}
int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
}
int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
}
int32_t compareInt8Val(const void *pLeft, const void *pRight) {
int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
return compareInt8Val(pRight, pLeft);
}
int32_t compareInt16Val(const void *pLeft, const void *pRight) {
int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
return compareInt16Val(pRight, pLeft);
}
int32_t compareInt32Val(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
return compareInt32Val(pRight, pLeft);
}
int32_t compareInt64Val(const void *pLeft, const void *pRight) {
int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
return compareInt64Val(pRight, pLeft);
}
int32_t compareUint32Val(const void *pLeft, const void *pRight) {
uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
return compareUint32Val(pRight, pLeft);
}
int32_t compareUint64Val(const void *pLeft, const void *pRight) {
uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
return compareUint64Val(pRight, pLeft);
}
int32_t compareUint16Val(const void *pLeft, const void *pRight) {
uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
return compareUint16Val(pRight, pLeft);
}
int32_t compareUint8Val(const void* pLeft, const void* pRight) {
uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
return compareUint8Val(pRight, pLeft);
}
int32_t compareFloatVal(const void *pLeft, const void *pRight) {
float p1 = GET_FLOAT_VAL(pLeft);
float p2 = GET_FLOAT_VAL(pRight);
if (isnan(p1) && isnan(p2)) {
return 0;
}
if (isnan(p1)) {
return -1;
}
if (isnan(p2)) {
return 1;
}
if (FLT_EQUAL(p1, p2)) {
return 0;
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
return compareFloatVal(pRight, pLeft);
}
int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
double p1 = GET_DOUBLE_VAL(pLeft);
double p2 = GET_DOUBLE_VAL(pRight);
if (isnan(p1) && isnan(p2)) {
return 0;
}
if (isnan(p1)) {
return -1;
}
if (isnan(p2)) {
return 1;
}
if (FLT_EQUAL(p1, p2)) {
return 0;
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
return compareDoubleVal(pRight, pLeft);
}
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
int32_t len1 = varDataLen(pLeft);
int32_t len2 = varDataLen(pRight);
if (len1 != len2) {
return len1 > len2? 1:-1;
} else {
int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1:-1;
}
}
}
int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
return compareLenPrefixedStr(pRight, pLeft);
}
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
int32_t len1 = varDataLen(pLeft);
int32_t len2 = varDataLen(pRight);
if (len1 != len2) {
return len1 > len2? 1:-1;
} else {
int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
}
int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
return compareLenPrefixedWStr(pRight, pLeft);
}
/*
* Compare two strings
* TSDB_MATCH: Match
* TSDB_NOMATCH: No match
* TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards.
* Like matching rules:
* '%': Matches zero or more characters
* '_': Matches one character
*
*/
int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
char c, c1;
int32_t i = 0;
int32_t j = 0;
int32_t o = 0;
int32_t m = 0;
while ((c = patterStr[i++]) != 0) {
if (c == pInfo->matchAll) { /* Match "*" */
while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
if (c == pInfo->matchOne) {
if (j > size || str[j++] == 0) {
// empty string, return not match
return TSDB_PATTERN_NOWILDCARDMATCH;
} else {
++o;
}
}
}
if (c == 0) {
return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
}
char next[3] = {toupper(c), tolower(c), 0};
m = o;
while (1) {
size_t n = strcspn(str + m, next);
str += m + n;
if (str[0] == 0 || (n >= size)) {
break;
}
int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
m = 0;
}
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
++o;
if (j <= size) {
if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; }
if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
continue;
}
}
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
wchar_t c, c1;
wchar_t matchOne = L'_'; // "_"
wchar_t matchAll = L'%'; // "%"
int32_t i = 0;
int32_t j = 0;
while ((c = patterStr[i++]) != 0) {
if (c == matchAll) { /* Match "%" */
while ((c = patterStr[i++]) == matchAll || c == matchOne) {
if (c == matchOne && (j >= size || str[j++] == 0)) {
return TSDB_PATTERN_NOWILDCARDMATCH;
}
}
if (c == 0) {
return TSDB_PATTERN_MATCH;
}
wchar_t accept[3] = {towupper(c), towlower(c), 0};
while (1) {
size_t n = wcscspn(str, accept);
str += n;
if (str[0] == 0 || (n >= size)) {
break;
}
int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
}
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
if (j <= size) {
if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
continue;
}
}
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'}; SPatternCompareInfo pInfo = {'%', '_'};
@ -359,62 +33,6 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
} }
int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) {
return compareStrRegexComp(pLeft, pRight);
}
int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) {
return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
}
int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
size_t sz = varDataLen(pRight);
char *pattern = malloc(sz + 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
pattern[sz] = 0;
sz = varDataLen(pLeft);
char *str = malloc(sz + 1);
memcpy(str, varDataVal(pLeft), sz);
str[sz] = 0;
int errCode = 0;
regex_t regex;
char msgbuf[256] = {0};
int cflags = REG_EXTENDED;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
regfree(&regex);
free(str);
free(pattern);
return 1;
}
errCode = regexec(&regex, str, 0, NULL, 0);
if (errCode != 0 && errCode != REG_NOMATCH) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
}
int32_t result = (errCode == 0) ? 0 : 1;
regfree(&regex);
free(str);
free(pattern);
return result;
}
int32_t taosArrayCompareString(const void* a, const void* b) {
const char* x = *(const char**)a;
const char* y = *(const char**)b;
return compareLenPrefixedStr(x, y);
}
int32_t compareFindItemInSet(const void *pLeft, const void* pRight) {
return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
}
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'}; SPatternCompareInfo pInfo = {'%', '_'};
@ -600,4 +218,4 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
} }
} }
} }
} }

77
source/common/src/trow.c
View File

@ -11,4 +11,79 @@
* *
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "trow.h"
/* ------------ Structures ---------- */
struct SRowBatch {
int32_t compress : 1; // if batch row is compressed
int32_t nrows : 31; // number of rows
int32_t tlen; // total length (including `nrows` and `tlen`)
char rows[];
};
struct SRowBuilder {
// TODO
};
struct SRowBatchIter {
int32_t counter; // row counter
SRowBatch *rb; // row batch to iter
SRow nrow; // next row
};
struct SRowBatchBuilder {
// TODO
};
/* ------------ Methods ---------- */
// SRowBuilder
SRowBuilder *rowBuilderCreate() {
SRowBuilder *pRowBuilder = NULL;
// TODO
return pRowBuilder;
}
void rowBuilderDestroy(SRowBuilder *pRowBuilder) {
if (pRowBuilder) {
free(pRowBuilder);
}
}
// SRowBatchIter
SRowBatchIter *rowBatchIterCreate(SRowBatch *pRowBatch) {
SRowBatchIter *pRowBatchIter = (SRowBatchIter *)malloc(sizeof(*pRowBatchIter));
if (pRowBatchIter == NULL) {
return NULL;
}
pRowBatchIter->counter = 0;
pRowBatchIter->rb = pRowBatch;
pRowBatchIter->nrow = pRowBatch->rows;
return pRowBatchIter;
};
void rowBatchIterDestroy(SRowBatchIter *pRowBatchIter) {
if (pRowBatchIter) {
free(pRowBatchIter);
}
}
const SRow rowBatchIterNext(SRowBatchIter *pRowBatchIter) {
SRow r = NULL;
if (pRowBatchIter->counter < pRowBatchIter->rb->nrows) {
r = pRowBatchIter->nrow;
pRowBatchIter->counter += 1;
pRowBatchIter->nrow = (SRow)POINTER_SHIFT(r, rowLen(r));
}
return r;
}
// SRowBatchBuilder
SRowBatchBuilder *rowBatchBuilderCreate();
void rowBatchBuilderDestroy(SRowBatchBuilder *);

1
source/server/CMakeLists.txt
View File

@ -8,4 +8,5 @@ add_executable(taosd ${TAOSD_SRC})
target_link_libraries( target_link_libraries(
taosd taosd
PUBLIC dnode PUBLIC dnode
PUBLIC util
) )

9
source/server/vnode/CMakeLists.txt
View File

@ -3,7 +3,7 @@ add_subdirectory(tq)
add_subdirectory(tsdb) add_subdirectory(tsdb)
aux_source_directory(src VNODE_SRC) aux_source_directory(src VNODE_SRC)
add_library(vnode ${VNODE_SRC}) add_library(vnode STATIC ${VNODE_SRC})
target_include_directories( target_include_directories(
vnode vnode
PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode" PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode"
@ -17,4 +17,9 @@ target_link_libraries(
PUBLIC tsdb PUBLIC tsdb
PUBLIC wal PUBLIC wal
PUBLIC cjson PUBLIC cjson
) )
# test
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})

2
source/server/vnode/inc/vnodeInt.h
View File

@ -79,7 +79,7 @@ typedef struct {
void * cq; // continuous query void * cq; // continuous query
int32_t dbCfgVersion; int32_t dbCfgVersion;
int32_t vgCfgVersion; int32_t vgCfgVersion;
STsdbCfg tsdbCfg; // STsdbCfg tsdbCfg;
#if 0 #if 0
SSyncCfg syncCfg; SSyncCfg syncCfg;
#endif #endif

28
source/server/vnode/inc/vnodeMemAllocator.h
View File

@ -16,12 +16,38 @@
#ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_
#define _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_
#include "amalloc.h" #include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct SVnodeMemAllocator SVnodeMemAllocator;
SVnodeMemAllocator *VMACreate(size_t size /* base size */, size_t ssize /* step size */,
size_t threshold /* threshold size when full*/);
void VMADestroy(SVnodeMemAllocator *pvma);
void VMAReset(SVnodeMemAllocator *pvma);
void * VMAMalloc(SVnodeMemAllocator *pvma, size_t size);
void VMAFree(SVnodeMemAllocator *pvma, void *ptr);
bool VMAIsFull(SVnodeMemAllocator *pvma);
// ------------------ FOR TEST ONLY ------------------
typedef struct SVMANode {
struct SVMANode *prev;
size_t tsize;
size_t used;
char data[];
} SVMANode;
struct SVnodeMemAllocator {
bool full; // if allocator is full
size_t threshold; // threshold;
size_t ssize; // step size to allocate
SVMANode *inuse; // inuse node to allocate
SVMANode node; // basic node to use
};
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

9
source/server/vnode/meta/CMakeLists.txt
View File

@ -1,5 +1,5 @@
aux_source_directory(src META_SRC) aux_source_directory(src META_SRC)
add_library(meta ${META_SRC}) add_library(meta STATIC ${META_SRC})
target_include_directories( target_include_directories(
meta meta
PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
@ -8,4 +8,9 @@ target_include_directories(
target_link_libraries( target_link_libraries(
meta meta
PUBLIC common PUBLIC common
) )
target_link_libraries(meta PUBLIC rocksdb)
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})

76
source/server/vnode/meta/src/meta.c
View File

@ -13,24 +13,72 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <rocksdb/c.h>
#include "thash.h"
#include "tlist.h"
#include "tlockfree.h"
#include "ttypes.h"
#include "meta.h" #include "meta.h"
int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) { typedef struct STable {
// TODO uint64_t uid;
return 0; tstr * name;
uint64_t suid;
SArray * schema;
} STable;
typedef struct STableObj {
bool pin;
uint64_t ref;
SRWLatch latch;
uint64_t offset;
SList * ctbList; // child table list
STable * pTable;
} STableObj;
struct SMeta {
pthread_rwlock_t rwLock;
SHashObj * pTableObjHash; // uid --> STableObj
SList * stbList; // super table list
rocksdb_t *tbnameDb; // tbname --> uid
rocksdb_t *tagDb; // uid --> tag
rocksdb_t *schemaDb;
size_t totalUsed;
};
SMeta *metaOpen(SMetaOptions *options) {
SMeta *pMeta = NULL;
char * err = NULL;
pMeta = (SMeta *)calloc(1, sizeof(*pMeta));
if (pMeta == NULL) {
return NULL;
}
pthread_rwlock_init(&(pMeta->rwLock), NULL);
pMeta->pTableObjHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
pMeta->stbList = tdListNew(sizeof(STableObj *));
// Open tbname DB
rocksdb_options_t *tbnameDbOptions = rocksdb_options_create();
pMeta->tbnameDb = rocksdb_open(tbnameDbOptions, "tbname_uid_db", &err);
// Open tag DB
pMeta->tagDb = rocksdb_open(tbnameDbOptions, "uid_tag_db", &err);
// Open schema DB
pMeta->schemaDb = rocksdb_open(tbnameDbOptions, "schema_db", &err);
return pMeta;
} }
int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) { void metaClose(SMeta *pMeta) {
// TODO // TODO
return 0;
} }
int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) { int metaCommit(SMeta *meta) { return 0; }
// TODO
return 0;
}
int metaCommit(SMeta *pMeta) {
// TODO
return 0;
}

18
source/server/vnode/meta/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
add_executable(metaTest "")
target_sources(metaTest
PRIVATE
"../src/meta.c"
"metaTests.cpp"
)
target_include_directories(metaTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(metaTest
os
util
common
rocksdb
gtest_main
)

9
source/server/vnode/meta/test/metaTests.cpp
View File

@ -0,0 +1,9 @@
#include <gtest/gtest.h>
#include <iostream>
#include "meta.h"
TEST(MetaTest, meta_open_test) {
metaOpen(NULL);
std::cout << "Hello META!" << std::endl;
}

112
source/server/vnode/src/vnodeMemAllocator.c
View File

@ -11,4 +11,114 @@
* *
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "vnodeMemAllocator.h"
#define VMA_IS_FULL(pvma) \
(((pvma)->inuse != &((pvma)->node)) || ((pvma)->inuse->tsize - (pvma)->inuse->used < (pvma)->threshold))
static SVMANode *VMANodeNew(size_t size);
static void VMANodeFree(SVMANode *node);
SVnodeMemAllocator *VMACreate(size_t size, size_t ssize, size_t threshold) {
SVnodeMemAllocator *pvma = NULL;
if (size < threshold) {
return NULL;
}
pvma = (SVnodeMemAllocator *)malloc(sizeof(*pvma) + size);
if (pvma) {
pvma->full = false;
pvma->threshold = threshold;
pvma->ssize = ssize;
pvma->inuse = &(pvma->node);
pvma->inuse->prev = NULL;
pvma->inuse->tsize = size;
pvma->inuse->used = 0;
}
return pvma;
}
void VMADestroy(SVnodeMemAllocator *pvma) {
if (pvma) {
VMAReset(pvma);
free(pvma);
}
}
void VMAReset(SVnodeMemAllocator *pvma) {
while (pvma->inuse != &(pvma->node)) {
SVMANode *node = pvma->inuse;
pvma->inuse = node->prev;
VMANodeFree(node);
}
pvma->inuse->used = 0;
pvma->full = false;
}
void *VMAMalloc(SVnodeMemAllocator *pvma, size_t size) {
void * ptr = NULL;
size_t tsize = size + sizeof(size_t);
if (pvma->inuse->tsize - pvma->inuse->used < tsize) {
SVMANode *pNode = VMANodeNew(MAX(pvma->ssize, tsize));
if (pNode == NULL) {
return NULL;
}
pNode->prev = pvma->inuse;
pvma->inuse = pNode;
}
ptr = pvma->inuse->data + pvma->inuse->used;
pvma->inuse->used += tsize;
*(size_t *)ptr = size;
ptr = POINTER_SHIFT(ptr, sizeof(size_t));
pvma->full = VMA_IS_FULL(pvma);
return ptr;
}
void VMAFree(SVnodeMemAllocator *pvma, void *ptr) {
if (ptr) {
size_t size = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t));
if (POINTER_SHIFT(ptr, size) == pvma->inuse->data + pvma->inuse->used) {
pvma->inuse->used -= (size + sizeof(size_t));
if ((pvma->inuse->used == 0) && (pvma->inuse != &(pvma->node))) {
SVMANode *node = pvma->inuse;
pvma->inuse = node->prev;
VMANodeFree(node);
}
pvma->full = VMA_IS_FULL(pvma);
}
}
}
bool VMAIsFull(SVnodeMemAllocator *pvma) { return pvma->full; }
static SVMANode *VMANodeNew(size_t size) {
SVMANode *node = NULL;
node = (SVMANode *)malloc(sizeof(*node) + size);
if (node) {
node->prev = NULL;
node->tsize = size;
node->used = 0;
}
return node;
}
static void VMANodeFree(SVMANode *node) {
if (node) {
free(node);
}
}

9
source/server/vnode/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,9 @@
# vnodeMemAllocatorTest
add_executable(VMATest "")
target_sources(VMATest
PRIVATE
"../src/vnodeMemAllocator.c"
"vnodeMemAllocatorTest.cpp"
)
target_include_directories(VMATest PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc")
target_link_libraries(VMATest os gtest_main)

22
source/server/vnode/test/vnodeMemAllocatorTest.cpp Normal file
View File

@ -0,0 +1,22 @@
#include <gtest/gtest.h>
#include <iostream>
#include "vnodeMemAllocator.h"
TEST(VMATest, basic_create_and_destroy_test) {
SVnodeMemAllocator *vma = VMACreate(1024, 512, 64);
EXPECT_TRUE(vma != nullptr);
EXPECT_EQ(vma->full, false);
EXPECT_EQ(vma->ssize, 512);
EXPECT_EQ(vma->threshold, 64);
EXPECT_EQ(vma->inuse->tsize, 1024);
VMADestroy(vma);
vma = VMACreate(1024, 512, 1024);
EXPECT_TRUE(vma != nullptr);
VMADestroy(vma);
vma = VMACreate(1024, 512, 1025);
EXPECT_TRUE(vma == nullptr);
VMADestroy(vma);
}

2
source/server/vnode/tsdb/src/tsdb.c
View File

@ -15,5 +15,5 @@
#include "tsdb.h" #include "tsdb.h"
int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg) { return 0; } int tsdbInsert(STsdb *tsdb, SSubmitReq *pReq, SSubmitRsp *pRsp) { return 0; }
int tsdbCommit(STsdb *pTsdb) { return 0; } int tsdbCommit(STsdb *pTsdb) { return 0; }

2
source/util/CMakeLists.txt
View File

@ -1,5 +1,5 @@
aux_source_directory(src UTIL_SRC) aux_source_directory(src UTIL_SRC)
add_library(util ${UTIL_SRC}) add_library(util STATIC ${UTIL_SRC})
target_include_directories( target_include_directories(
util util
PUBLIC "${CMAKE_SOURCE_DIR}/include/util" PUBLIC "${CMAKE_SOURCE_DIR}/include/util"

398
source/util/src/compare.c Normal file
View File

@ -0,0 +1,398 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _BSD_SOURCE
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "os.h"
#include "types.h"
#include "compare.h"
#include "ulog.h"
#include "thash.h"
#include "regex.h"
int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
}
int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
}
int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
}
int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
}
int32_t compareInt8Val(const void *pLeft, const void *pRight) {
int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
return compareInt8Val(pRight, pLeft);
}
int32_t compareInt16Val(const void *pLeft, const void *pRight) {
int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
return compareInt16Val(pRight, pLeft);
}
int32_t compareInt32Val(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
return compareInt32Val(pRight, pLeft);
}
int32_t compareInt64Val(const void *pLeft, const void *pRight) {
int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
return compareInt64Val(pRight, pLeft);
}
int32_t compareUint32Val(const void *pLeft, const void *pRight) {
uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
return compareUint32Val(pRight, pLeft);
}
int32_t compareUint64Val(const void *pLeft, const void *pRight) {
uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
return compareUint64Val(pRight, pLeft);
}
int32_t compareUint16Val(const void *pLeft, const void *pRight) {
uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
return compareUint16Val(pRight, pLeft);
}
int32_t compareUint8Val(const void* pLeft, const void* pRight) {
uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
return compareUint8Val(pRight, pLeft);
}
int32_t compareFloatVal(const void *pLeft, const void *pRight) {
float p1 = GET_FLOAT_VAL(pLeft);
float p2 = GET_FLOAT_VAL(pRight);
if (isnan(p1) && isnan(p2)) {
return 0;
}
if (isnan(p1)) {
return -1;
}
if (isnan(p2)) {
return 1;
}
if (FLT_EQUAL(p1, p2)) {
return 0;
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
return compareFloatVal(pRight, pLeft);
}
int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
double p1 = GET_DOUBLE_VAL(pLeft);
double p2 = GET_DOUBLE_VAL(pRight);
if (isnan(p1) && isnan(p2)) {
return 0;
}
if (isnan(p1)) {
return -1;
}
if (isnan(p2)) {
return 1;
}
if (FLT_EQUAL(p1, p2)) {
return 0;
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
return compareDoubleVal(pRight, pLeft);
}
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
int32_t len1 = varDataLen(pLeft);
int32_t len2 = varDataLen(pRight);
if (len1 != len2) {
return len1 > len2? 1:-1;
} else {
int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1:-1;
}
}
}
int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
return compareLenPrefixedStr(pRight, pLeft);
}
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
int32_t len1 = varDataLen(pLeft);
int32_t len2 = varDataLen(pRight);
if (len1 != len2) {
return len1 > len2? 1:-1;
} else {
int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
}
int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
return compareLenPrefixedWStr(pRight, pLeft);
}
/*
* Compare two strings
* TSDB_MATCH: Match
* TSDB_NOMATCH: No match
* TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards.
* Like matching rules:
* '%': Matches zero or more characters
* '_': Matches one character
*
*/
int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
char c, c1;
int32_t i = 0;
int32_t j = 0;
int32_t o = 0;
int32_t m = 0;
while ((c = patterStr[i++]) != 0) {
if (c == pInfo->matchAll) { /* Match "*" */
while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
if (c == pInfo->matchOne) {
if (j > size || str[j++] == 0) {
// empty string, return not match
return TSDB_PATTERN_NOWILDCARDMATCH;
} else {
++o;
}
}
}
if (c == 0) {
return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
}
char next[3] = {toupper(c), tolower(c), 0};
m = o;
while (1) {
size_t n = strcspn(str + m, next);
str += m + n;
if (str[0] == 0 || (n >= size)) {
break;
}
int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
m = 0;
}
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
++o;
if (j <= size) {
if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; }
if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
continue;
}
}
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
wchar_t c, c1;
wchar_t matchOne = L'_'; // "_"
wchar_t matchAll = L'%'; // "%"
int32_t i = 0;
int32_t j = 0;
while ((c = patterStr[i++]) != 0) {
if (c == matchAll) { /* Match "%" */
while ((c = patterStr[i++]) == matchAll || c == matchOne) {
if (c == matchOne && (j >= size || str[j++] == 0)) {
return TSDB_PATTERN_NOWILDCARDMATCH;
}
}
if (c == 0) {
return TSDB_PATTERN_MATCH;
}
wchar_t accept[3] = {towupper(c), towlower(c), 0};
while (1) {
size_t n = wcscspn(str, accept);
str += n;
if (str[0] == 0 || (n >= size)) {
break;
}
int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
}
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
if (j <= size) {
if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
continue;
}
}
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) {
return compareStrRegexComp(pLeft, pRight);
}
int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) {
return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
}
int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
size_t sz = varDataLen(pRight);
char *pattern = malloc(sz + 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
pattern[sz] = 0;
sz = varDataLen(pLeft);
char *str = malloc(sz + 1);
memcpy(str, varDataVal(pLeft), sz);
str[sz] = 0;
int errCode = 0;
regex_t regex;
char msgbuf[256] = {0};
int cflags = REG_EXTENDED;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
regfree(&regex);
free(str);
free(pattern);
return 1;
}
errCode = regexec(&regex, str, 0, NULL, 0);
if (errCode != 0 && errCode != REG_NOMATCH) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
}
int32_t result = (errCode == 0) ? 0 : 1;
regfree(&regex);
free(str);
free(pattern);
return result;
}
int32_t taosArrayCompareString(const void* a, const void* b) {
const char* x = *(const char**)a;
const char* y = *(const char**)b;
return compareLenPrefixedStr(x, y);
}
int32_t compareFindItemInSet(const void *pLeft, const void* pRight) {
return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
}

0
source/common/src/terror.c → source/util/src/terror.c
View File

3
source/util/src/thashutil.c
View File

@ -15,8 +15,9 @@
#include "os.h" #include "os.h"
#include "thash.h" #include "thash.h"
#include "tcompare.h" #include "compare.h"
#include "tdef.h" #include "tdef.h"
#include "types.h"
#define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r))) #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r)))

2
source/util/src/tskiplist.c
View File

@ -15,7 +15,7 @@
*/ */
#include "tskiplist.h" #include "tskiplist.h"
#include "os.h" #include "os.h"
#include "tcompare.h" #include "compare.h"
#include "ulog.h" #include "ulog.h"
#include "tutil.h" #include "tutil.h"

21
source/util/src/tutil.c
View File

@ -450,24 +450,3 @@ char *taosIpStr(uint32_t ipInt) {
return ipStr; return ipStr;
} }
FORCE_INLINE float taos_align_get_float(const char* pBuf) {
#if __STDC_VERSION__ >= 201112L
static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
#else
assert(sizeof(float) == sizeof(uint32_t));
#endif
float fv = 0;
memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem
return fv;
}
FORCE_INLINE double taos_align_get_double(const char* pBuf) {
#if __STDC_VERSION__ >= 201112L
static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
#else
assert(sizeof(double) == sizeof(uint64_t));
#endif
double dv = 0;
memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem
return dv;
}