xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/dnode3

This commit is contained in:
Shengliang Guan 2021-10-12 17:08:57 +08:00
parent 8519828e31 cbe172eb37
commit c699fe7c1c
46 changed files with 1456 additions and 647 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CMakeLists.txt
View File

@ -35,6 +35,18 @@ cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${DEPS_TMP_FILE})
## cJson ## cJson
cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE}) cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE})
## leveldb
option(BUILD_WITH_LEVELDB "If build with leveldb" OFF)
if(${BUILD_WITH_LEVELDB})
cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
endif(${BUILD_WITH_LEVELDB})
## rocksdb
option(BUILD_WITH_ROCKSDB "If build with rocksdb" OFF)
if(${BUILD_WITH_ROCKSDB})
cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
endif(${BUILD_WITH_ROCKSDB})
## download dependencies ## download dependencies
configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt") configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .

1
cmake/EnableCMP0048.txt.in Normal file
View File

@ -0,0 +1 @@
cmake_policy(SET CMP0048 NEW)

13
cmake/leveldb_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,13 @@
# leveldb
ExternalProject_Add(leveldb
GIT_REPOSITORY https://github.com/taosdata-contrib/leveldb.git
GIT_TAG master
SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/leveldb"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

11
cmake/rocksdb_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,11 @@
# rocksdb
ExternalProject_Add(rocksdb
GIT_REPOSITORY https://github.com/taosdata-contrib/rocksdb.git
GIT_TAG v6.23.3
SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/rocksdb"
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

34
deps/CMakeLists.txt vendored
View File

@ -1,23 +1,57 @@
# googletest
if(${BUILD_TEST}) if(${BUILD_TEST})
add_subdirectory(googletest) add_subdirectory(googletest)
endif(${BUILD_TEST}) endif(${BUILD_TEST})
# cJson
# see https://stackoverflow.com/questions/37582508/silence-cmp0048-warnings-in-vendored-projects
set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
add_subdirectory(cJson) add_subdirectory(cJson)
target_include_directories( target_include_directories(
cjson cjson
# see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cJson> PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cJson>
) )
unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# lz4
add_subdirectory(lz4/build/cmake) add_subdirectory(lz4/build/cmake)
target_include_directories( target_include_directories(
lz4_static lz4_static
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
) )
# zlib
set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
add_subdirectory(zlib) add_subdirectory(zlib)
target_include_directories( target_include_directories(
zlib zlib
PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib
) )
unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# leveldb
if(${BUILD_WITH_LEVELDB})
option(LEVELDB_BUILD_TESTS "" OFF)
add_subdirectory(leveldb)
target_include_directories(
leveldb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
)
endif(${BUILD_WITH_LEVELDB})
# rocksdb
if(${BUILD_WITH_ROCKSDB})
option(WITH_TESTS "" OFF)
option(WITH_BENCHMARK_TOOLS "" OFF)
option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
add_subdirectory(rocksdb)
target_include_directories(
rocksdb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
)
endif(${BUILD_WITH_ROCKSDB})

103
include/common/schema.h
View File

@ -1,103 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_SCHEMA_H_
#define _TD_COMMON_SCHEMA_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
// ----------------- TSDB COLUMN DEFINITION
typedef struct {
int8_t type; // Column type
int16_t colId; // column ID
int16_t bytes; // column bytes (restore to int16_t in case of misuse)
uint16_t offset; // point offset in SDataRow after the header part.
} STColumn;
#define colType(col) ((col)->type)
#define colColId(col) ((col)->colId)
#define colBytes(col) ((col)->bytes)
#define colOffset(col) ((col)->offset)
#define colSetType(col, t) (colType(col) = (t))
#define colSetColId(col, id) (colColId(col) = (id))
#define colSetBytes(col, b) (colBytes(col) = (b))
#define colSetOffset(col, o) (colOffset(col) = (o))
// ----------------- TSDB SCHEMA DEFINITION
typedef struct {
int version; // version
int numOfCols; // Number of columns appended
int tlen; // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + //
// (bytes))
uint16_t flen; // First part length in a SDataRow after the header part
uint16_t vlen; // pure value part length, excluded the overhead (bytes only)
STColumn columns[];
} STSchema;
#define schemaNCols(s) ((s)->numOfCols)
#define schemaVersion(s) ((s)->version)
#define schemaTLen(s) ((s)->tlen)
#define schemaFLen(s) ((s)->flen)
#define schemaVLen(s) ((s)->vlen)
#define schemaColAt(s, i) ((s)->columns + i)
#define tdFreeSchema(s) tfree((s))
STSchema *tdDupSchema(STSchema *pSchema);
int tdEncodeSchema(void **buf, STSchema *pSchema);
void * tdDecodeSchema(void *buf, STSchema **pRSchema);
static FORCE_INLINE int comparColId(const void *key1, const void *key2) {
if (*(int16_t *)key1 > ((STColumn *)key2)->colId) {
return 1;
} else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) {
return -1;
} else {
return 0;
}
}
static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) {
void *ptr = bsearch(&colId, (void *)pSchema->columns, schemaNCols(pSchema), sizeof(STColumn), comparColId);
if (ptr == NULL) return NULL;
return (STColumn *)ptr;
}
// ----------------- SCHEMA BUILDER DEFINITION
typedef struct {
int tCols;
int nCols;
int tlen;
uint16_t flen;
uint16_t vlen;
int version;
STColumn *columns;
} STSchemaBuilder;
int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes);
STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
#ifdef __cplusplus
}
#endif
#endif /*_TD_COMMON_SCHEMA_H_*/

36
include/common/tcompare.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TCOMPARE_H_
#define _TD_TCOMPARE_H_
#include "compare.h"
#include "ttypes.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);
__compar_fn_t getComparFunc(int32_t type, int32_t optr);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TCOMPARE_H_*/

2
include/common/tdataformat.h
View File

@ -87,7 +87,7 @@ typedef struct {
#define schemaColAt(s, i) ((s)->columns + i) #define schemaColAt(s, i) ((s)->columns + i)
#define tdFreeSchema(s) tfree((s)) #define tdFreeSchema(s) tfree((s))
STSchema *tdDupSchema(STSchema *pSchema); STSchema *tdDupSchema(const STSchema *pSchema);
int tdEncodeSchema(void **buf, STSchema *pSchema); int tdEncodeSchema(void **buf, STSchema *pSchema);
void * tdDecodeSchema(void *buf, STSchema **pRSchema); void * tdDecodeSchema(void *buf, STSchema **pRSchema);

45
include/common/trow.h
View File

@ -16,10 +16,55 @@
#ifndef _TD_COMMON_ROW_H_ #ifndef _TD_COMMON_ROW_H_
#define _TD_COMMON_ROW_H_ #define _TD_COMMON_ROW_H_
#include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
// types
typedef void * SRow;
typedef struct SRowBatch SRowBatch;
typedef struct SRowBuilder SRowBuilder;
typedef struct SRowBatchIter SRowBatchIter;
typedef struct SRowBatchBuilder SRowBatchBuilder;
// SRow
#define ROW_HEADER_SIZE (sizeof(uint8_t) + 2 * sizeof(uint16_t) + sizeof(uint64_t))
#define rowType(r) (*(uint8_t *)(r)) // row type
#define rowLen(r) (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t))) // row length
#define rowSVer(r) \
(*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t) + sizeof(uint16_t))) // row schema version, only for SDataRow
#define rowNCols(r) rowSVer(r) // only for SKVRow
#define rowVer(r) (*(uint64_t)POINTER_SHIFT(r, sizeof(uint8_t) + 2 * sizeof(uint16_t))) // row version
#define rowCopy(dest, r) memcpy((dest), r, rowLen(r))
static FORCE_INLINE SRow rowDup(SRow row) {
SRow r = malloc(rowLen(row));
if (r == NULL) {
return NULL;
}
rowCopy(r, row);
return r;
}
// SRowBatch
// SRowBuilder
SRowBuilder *rowBuilderCreate();
void rowBuilderDestroy(SRowBuilder *);
// SRowBatchIter
SRowBatchIter *rowBatchIterCreate(SRowBatch *);
void rowBatchIterDestroy(SRowBatchIter *);
const SRow rowBatchIterNext(SRowBatchIter *);
// SRowBatchBuilder
SRowBatchBuilder *rowBatchBuilderCreate();
void rowBatchBuilderDestroy(SRowBatchBuilder *);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

27
include/common/tschema.h Normal file
View File

@ -0,0 +1,27 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_SCHEMA_H_
#define _TD_COMMON_SCHEMA_H_
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_COMMON_SCHEMA_H_*/

6
include/common/ttypes.h
View File

@ -6,10 +6,10 @@ extern "C" {
#endif #endif
#include "taosdef.h" #include "taosdef.h"
#include "types.h"
// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
typedef int32_t VarDataOffsetT; typedef int32_t VarDataOffsetT;
typedef int16_t VarDataLenT; // maxVarDataLen: 32767
typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535 typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535
typedef uint32_t TDRowTLenT; // total length, including overhead typedef uint32_t TDRowTLenT; // total length, including overhead
@ -30,11 +30,7 @@ typedef struct {
} SNCharNullT; } SNCharNullT;
#pragma pack(pop) #pragma pack(pop)
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
#define varDataLen(v) ((VarDataLenT *)(v))[0]
#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) #define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) #define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))

68
include/libs/tkv/tkv.h
View File

@ -16,56 +16,42 @@
#ifndef _TD_TKV_H_ #ifndef _TD_TKV_H_
#define _TD_TKV_H_ #define _TD_TKV_H_
#include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct tkv_db_s tkv_db_t; // Types exported
typedef struct STkvDb STkvDb;
typedef struct STkvOpts STkvOpts;
typedef struct STkvCache STkvCache;
typedef struct STkvReadOpts STkvReadOpts;
typedef struct STkvWriteOpts STkvWriteOpts;
typedef struct { // DB operations
/* data */ STkvDb *tkvOpen(const STkvOpts *options, const char *path);
} tkv_key_t; void tkvClose(STkvDb *db);
void tkvPut(STkvDb *db, STkvWriteOpts *, char *key, size_t keylen, char *val, size_t vallen);
char * tkvGet(STkvDb *db, STkvReadOpts *, char *key, size_t keylen, size_t *vallen);
typedef struct { // DB options
bool pinned; STkvOpts *tkvOptionsCreate();
int64_t ref; // TODO: use util library void tkvOptionsDestroy(STkvOpts *);
// TODO: add a RW latch here void tkvOptionsSetCache(STkvOpts *, STkvCache *);
uint64_t offset;
void * pObj;
} tkv_obj_t;
typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *); // DB cache
typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *); typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType;
typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj); STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type);
typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj); void tkvCacheDestroy(STkvCache *);
typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *);
typedef void (*tkv_obj_destroy_fn_t)(void *);
typedef struct { // STkvReadOpts
uint64_t memLimit; STkvReadOpts *tkvReadOptsCreate();
tkv_get_key_fn_t getKey; void tkvReadOptsDestroy(STkvReadOpts *);
tkv_obj_encode_fn_t encode;
tkv_obj_decode_fn_t decode;
tkv_obj_comp_fn_t compare;
tkv_obj_destroy_fn_t destroy;
} tkv_db_option_t;
tkv_db_t * tkvOpenDB(char *dir, tkv_db_option_t *); // STkvWriteOpts
int tkvCloseDB(tkv_db_t *); STkvWriteOpts *tkvWriteOptsCreate();
int tkvPut(tkv_db_t *, tkv_obj_t *); void tkvWriteOptsDestroy(STkvWriteOpts *);
int tkvPutBatch(tkv_db_t *, tkv_obj_t **, int); // TODO: use array here
const tkv_obj_t *tkvGet(tkv_key_t *);
int tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **); // TODO: use array here
int tkvDrop(tkv_db_t *, tkv_key_t *);
int tkvDropBatch(tkv_db_t *, tkv_key_t **, int); // TODO: use array here
int tkvCommit(tkv_db_t *, void * /*TODO*/);
typedef struct {
} tkv_db_iter_t;
tkv_db_iter_t * tkvIterNew(tkv_db_t *);
void tkvIterFree(tkv_db_iter_t *);
const tkv_obj_t *tkvIterNext(tkv_db_iter_t *);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
include/os/osDir.h
View File

@ -20,7 +20,7 @@
extern "C" { extern "C" {
#endif #endif
void taosRemoveDir(char *dirname); void taosRemoveDir(const char *dirname);
bool taosDirExist(char *dirname); bool taosDirExist(char *dirname);
bool taosMkDir(char *dirname); bool taosMkDir(char *dirname);
void taosRemoveOldFiles(char *dirname, int32_t keepDays); void taosRemoveOldFiles(char *dirname, int32_t keepDays);

47
include/server/vnode/meta/meta.h
View File

@ -18,16 +18,53 @@
#include "taosmsg.h" #include "taosmsg.h"
#include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct SMeta SMeta; typedef uint64_t tuid_t;
int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq); // Types exported
int metaDropTable(SMeta *pMeta, SDropTableReq *pReq); typedef struct SMeta SMeta;
int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq); typedef struct SMetaOpts SMetaOpts;
int metaCommit(SMeta *pMeta); typedef struct SMetaQueryHandle SMetaQueryHandle;
typedef struct SMetaQueryOpts SMetaQueryOpts;
typedef struct STableOpts STableOpts;
// SMeta operations
int metaCreate(const char *path);
void metaDestroy(const char *path);
SMeta *metaOpen(SMetaOpts *);
void metaClose(SMeta *);
int metaCreateTable(SMeta *, STableOpts *);
int metaDropTable(SMeta *, uint64_t tuid_t);
int metaAlterTable(SMeta *, void *);
int metaCommit(SMeta *);
// Options
SMetaOpts *metaOptionsCreate();
void metaOptionsDestroy(SMetaOpts *);
void metaOptionsSetCache(SMetaOpts *, size_t capacity);
// SMetaQueryHandle
SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOpts *);
void metaQueryHandleDestroy(SMetaQueryHandle *);
// SMetaQueryOpts
SMetaQueryOpts *metaQueryOptionsCreate();
void metaQueryOptionsDestroy(SMetaQueryOpts *);
// STableOpts
void metaTableOptsInit(STableOpts *, int8_t type, const char *name, const STSchema *pSchema);
/* -------------------------------- Hided implementations -------------------------------- */
struct STableOpts {
int8_t type;
char * name;
STSchema *pSchema;
};
#ifdef __cplusplus #ifdef __cplusplus
} }

75
include/server/vnode/tsdb/tsdb.h
View File

@ -23,33 +23,60 @@
extern "C" { extern "C" {
#endif #endif
typedef struct STsdb STsdb; // Types exported
typedef struct { typedef struct STsdb STsdb;
int32_t id; // TODO: use a global definition typedef struct STsdbOptions STsdbOptions;
int32_t days; typedef struct STsdbSMAOptions STsdbSMAOptions; // SMA stands for Small Materialized Aggregation
int32_t keep; typedef struct STsdbReadOptions STsdbReadOptions;
int32_t keep1; typedef struct STsdbSnapshot STsdbSnapshot;
int32_t keep2; typedef struct STsdbQueryHandle STsdbQueryHandle;
int32_t minRows;
int32_t maxRows;
int8_t precision;
int8_t update;
} STsdbCfg;
// Module init and clear // DB operations
int tsdbInit(); int tsdbCreate(const char *path);
int tsdbClear(); int tsdbDestroy(const char *path);
STsdb *tsdbOpen(const STsdbOptions *options);
void tsdbClose(STsdb *);
int tsdbReset(STsdb *, const STsdbOptions *);
int tsdbInsert(STsdb *, SSubmitReq *, SSubmitRsp *);
int tsdbCommit(STsdb *);
int tsdbCompact(STsdb *);
// Repository operations // Options
int tsdbCreateRepo(int id); STsdbOptions *tsdbOptionsCreate();
int tsdbDropRepo(int id); void tsdbOptionsDestroy(STsdbOptions *);
STsdb *tsdbOpenRepo(STsdbCfg *pCfg); void tsdbOptionsSetId(STsdbOptions *, int id);
int tsdbCloseRepo(STsdb *pTsdb); void tsdbOptionsSetHoursPerFile(STsdbOptions *, int hours);
int tsdbForceCloseRepo(STsdb *pTsdb); void tsdbOptionsSetRetention(STsdbOptions *, int keep, int keep1, int keep2);
void tsdbOptionsSetMinAndMaxRows(STsdbOptions *, int minRows, int maxRows);
void tsdbOptionsSetPrecision(STsdbOptions *, int);
void tsdbOptionsSetCache(STsdbOptions *, int);
typedef enum { TSDB_NO_UPDATE = 0, TSDB_WHOLE_ROW_UPDATE = 1, TSDB_PARTIAL_ROW_UPDATE = 2 } ETsdbUpdateType;
void tsdbOptionsSetUpdate(STsdbOptions *, ETsdbUpdateType);
void tsdbOptionsSetSMA(STsdbOptions *, STsdbSMAOptions *);
// Data commit // STsdbSMAOptions
int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg); STsdbSMAOptions *tsdbSMAOptionsCreate();
int tsdbCommit(STsdb *pTsdb); void tsdbSMAOptionsDestroy(STsdbSMAOptions *);
// void tsdbSMAOptionsSetFuncs(STsdbSMAOptions *, SArray * /*Array of function to perform on each block*/);
// void tsdbSMAOptionsSetIntervals(STsdbSMAOptions *, SArray *);
// void tsdbSMAOptionsSetColTypes(STsdbSMAOptions *, SArray *);
// STsdbQueryHandle
STsdbQueryHandle *tsdbQueryHandleCreate(STsdb *, STsdbReadOptions *);
void tsdbQueryHandleDestroy(STsdbQueryHandle *);
void tsdbResetQueryHandle(STsdbQueryHandle *, STsdbReadOptions *);
bool tsdbNextDataBlock(STsdbQueryHandle *);
// void tsdbGetDataBlockInfo(STsdbQueryHandle *, SDataBlockInfo *);
// void tsdbGetDataBlockStatisInfo(STsdbQueryHandle *, SDataStatis **);
// STsdbReadOptions
STsdbReadOptions *tsdbReadOptionsCreate();
void tsdbReadOptionsDestroy(STsdbReadOptions *);
void tsdbReadOptionsSetSnapshot(STsdbReadOptions *, STsdbSnapshot *);
// STsdbSnapshot
STsdbSnapshot *tsdbSnapshotCreate(STsdb *);
void tsdbSnapshotDestroy(STsdbSnapshot *);
#ifdef __cplusplus #ifdef __cplusplus
} }

19
include/util/tcompare.h → include/util/compare.h
View File

@ -46,11 +46,6 @@ int patternMatch(const char *pattern, const char *str, size_t size, const SPatte
int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
__compar_fn_t getComparFunc(int32_t type, int32_t optr);
int32_t taosArrayCompareString(const void* a, const void* b); int32_t taosArrayCompareString(const void* a, const void* b);
@ -82,12 +77,22 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
int32_t compareStrRegexComp(const void* pLeft, const void* pRight); int32_t compareStrRegexComp(const void* pLeft, const void* pRight);
int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight); int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight);
int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight); int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight);
int32_t compareFindItemInSet(const void *pLeft, const void* pRight); int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
int32_t compareInt16ValDesc(const void* pLeft, const void* pRight);
int32_t compareInt32ValDesc(const void* pLeft, const void* pRight);
int32_t compareInt64ValDesc(const void* pLeft, const void* pRight);
int32_t compareFloatValDesc(const void* pLeft, const void* pRight);
int32_t compareDoubleValDesc(const void* pLeft, const void* pRight);
int32_t compareUint8ValDesc(const void* pLeft, const void* pRight);
int32_t compareUint16ValDesc(const void* pLeft, const void* pRight);
int32_t compareUint32ValDesc(const void* pLeft, const void* pRight);
int32_t compareUint64ValDesc(const void* pLeft, const void* pRight);
int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight);
int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight);
#ifdef __cplusplus #ifdef __cplusplus
} }

27
include/util/tdef.h
View File

@ -110,33 +110,6 @@ do { \
(src) = (void *)((char *)src + sizeof(type));\ (src) = (void *)((char *)src + sizeof(type));\
} while(0) } while(0)
#define GET_INT8_VAL(x) (*(int8_t *)(x))
#define GET_INT16_VAL(x) (*(int16_t *)(x))
#define GET_INT32_VAL(x) (*(int32_t *)(x))
#define GET_INT64_VAL(x) (*(int64_t *)(x))
#define GET_UINT8_VAL(x) (*(uint8_t*) (x))
#define GET_UINT16_VAL(x) (*(uint16_t *)(x))
#define GET_UINT32_VAL(x) (*(uint32_t *)(x))
#define GET_UINT64_VAL(x) (*(uint64_t *)(x))
#ifdef _TD_ARM_32
float taos_align_get_float(const char* pBuf);
double taos_align_get_double(const char* pBuf);
#define GET_FLOAT_VAL(x) taos_align_get_float(x)
#define GET_DOUBLE_VAL(x) taos_align_get_double(x)
#define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); }
#define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
#define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); }
#define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
#else
#define GET_FLOAT_VAL(x) (*(float *)(x))
#define GET_DOUBLE_VAL(x) (*(double *)(x))
#define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); }
#define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); }
#define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); }
#define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
#endif
// TODO: check if below is necessary // TODO: check if below is necessary
#define TSDB_RELATION_INVALID 0 #define TSDB_RELATION_INVALID 0

88
include/util/types.h Normal file
View File

@ -0,0 +1,88 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TYPES_H_
#define _TD_TYPES_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
#define GET_INT8_VAL(x) (*(int8_t *)(x))
#define GET_INT16_VAL(x) (*(int16_t *)(x))
#define GET_INT32_VAL(x) (*(int32_t *)(x))
#define GET_INT64_VAL(x) (*(int64_t *)(x))
#define GET_UINT8_VAL(x) (*(uint8_t*) (x))
#define GET_UINT16_VAL(x) (*(uint16_t *)(x))
#define GET_UINT32_VAL(x) (*(uint32_t *)(x))
#define GET_UINT64_VAL(x) (*(uint64_t *)(x))
static FORCE_INLINE float taos_align_get_float(const char *pBuf) {
#if __STDC_VERSION__ >= 201112L
static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
#else
assert(sizeof(float) == sizeof(uint32_t));
#endif
float fv = 0;
memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem
return fv;
}
static FORCE_INLINE double taos_align_get_double(const char *pBuf) {
#if __STDC_VERSION__ >= 201112L
static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
#else
assert(sizeof(double) == sizeof(uint64_t));
#endif
double dv = 0;
memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem
return dv;
}
// #ifdef _TD_ARM_32
// float taos_align_get_float(const char* pBuf);
// double taos_align_get_double(const char* pBuf);
// #define GET_FLOAT_VAL(x) taos_align_get_float(x)
// #define GET_DOUBLE_VAL(x) taos_align_get_double(x)
// #define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); }
// #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
// #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); }
// #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
// #else
#define GET_FLOAT_VAL(x) (*(float *)(x))
#define GET_DOUBLE_VAL(x) (*(double *)(x))
#define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); }
#define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); }
#define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); }
#define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
// #endif
typedef int16_t VarDataLenT; // maxVarDataLen: 32767
#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
#define varDataLen(v) ((VarDataLenT *)(v))[0]
#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
typedef int32_t VarDataOffsetT;
typedef int16_t VarDataLenT; // maxVarDataLen: 32767
#ifdef __cplusplus
}
#endif
#endif /*_TD_TYPES_H_*/

384
source/common/src/tcompare.c
View File

@ -12,334 +12,8 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#define _BSD_SOURCE
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "os.h"
#include "tcompare.h" #include "tcompare.h"
#include "ulog.h"
#include "thash.h"
#include "regex.h"
#include "ttypes.h"
int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
}
int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
}
int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
}
int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
}
int32_t compareInt8Val(const void *pLeft, const void *pRight) {
int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
return compareInt8Val(pRight, pLeft);
}
int32_t compareInt16Val(const void *pLeft, const void *pRight) {
int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
return compareInt16Val(pRight, pLeft);
}
int32_t compareInt32Val(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
return compareInt32Val(pRight, pLeft);
}
int32_t compareInt64Val(const void *pLeft, const void *pRight) {
int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
return compareInt64Val(pRight, pLeft);
}
int32_t compareUint32Val(const void *pLeft, const void *pRight) {
uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
return compareUint32Val(pRight, pLeft);
}
int32_t compareUint64Val(const void *pLeft, const void *pRight) {
uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
return compareUint64Val(pRight, pLeft);
}
int32_t compareUint16Val(const void *pLeft, const void *pRight) {
uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
return compareUint16Val(pRight, pLeft);
}
int32_t compareUint8Val(const void* pLeft, const void* pRight) {
uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
return compareUint8Val(pRight, pLeft);
}
int32_t compareFloatVal(const void *pLeft, const void *pRight) {
float p1 = GET_FLOAT_VAL(pLeft);
float p2 = GET_FLOAT_VAL(pRight);
if (isnan(p1) && isnan(p2)) {
return 0;
}
if (isnan(p1)) {
return -1;
}
if (isnan(p2)) {
return 1;
}
if (FLT_EQUAL(p1, p2)) {
return 0;
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
return compareFloatVal(pRight, pLeft);
}
int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
double p1 = GET_DOUBLE_VAL(pLeft);
double p2 = GET_DOUBLE_VAL(pRight);
if (isnan(p1) && isnan(p2)) {
return 0;
}
if (isnan(p1)) {
return -1;
}
if (isnan(p2)) {
return 1;
}
if (FLT_EQUAL(p1, p2)) {
return 0;
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
return compareDoubleVal(pRight, pLeft);
}
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
int32_t len1 = varDataLen(pLeft);
int32_t len2 = varDataLen(pRight);
if (len1 != len2) {
return len1 > len2? 1:-1;
} else {
int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1:-1;
}
}
}
int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
return compareLenPrefixedStr(pRight, pLeft);
}
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
int32_t len1 = varDataLen(pLeft);
int32_t len2 = varDataLen(pRight);
if (len1 != len2) {
return len1 > len2? 1:-1;
} else {
int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
}
int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
return compareLenPrefixedWStr(pRight, pLeft);
}
/*
* Compare two strings
* TSDB_MATCH: Match
* TSDB_NOMATCH: No match
* TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards.
* Like matching rules:
* '%': Matches zero or more characters
* '_': Matches one character
*
*/
int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
char c, c1;
int32_t i = 0;
int32_t j = 0;
int32_t o = 0;
int32_t m = 0;
while ((c = patterStr[i++]) != 0) {
if (c == pInfo->matchAll) { /* Match "*" */
while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
if (c == pInfo->matchOne) {
if (j > size || str[j++] == 0) {
// empty string, return not match
return TSDB_PATTERN_NOWILDCARDMATCH;
} else {
++o;
}
}
}
if (c == 0) {
return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
}
char next[3] = {toupper(c), tolower(c), 0};
m = o;
while (1) {
size_t n = strcspn(str + m, next);
str += m + n;
if (str[0] == 0 || (n >= size)) {
break;
}
int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
m = 0;
}
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
++o;
if (j <= size) {
if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; }
if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
continue;
}
}
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
wchar_t c, c1;
wchar_t matchOne = L'_'; // "_"
wchar_t matchAll = L'%'; // "%"
int32_t i = 0;
int32_t j = 0;
while ((c = patterStr[i++]) != 0) {
if (c == matchAll) { /* Match "%" */
while ((c = patterStr[i++]) == matchAll || c == matchOne) {
if (c == matchOne && (j >= size || str[j++] == 0)) {
return TSDB_PATTERN_NOWILDCARDMATCH;
}
}
if (c == 0) {
return TSDB_PATTERN_MATCH;
}
wchar_t accept[3] = {towupper(c), towlower(c), 0};
while (1) {
size_t n = wcscspn(str, accept);
str += n;
if (str[0] == 0 || (n >= size)) {
break;
}
int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
}
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
if (j <= size) {
if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
continue;
}
}
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'}; SPatternCompareInfo pInfo = {'%', '_'};
@ -359,62 +33,6 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
} }
int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) {
return compareStrRegexComp(pLeft, pRight);
}
int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) {
return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
}
int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
size_t sz = varDataLen(pRight);
char *pattern = malloc(sz + 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
pattern[sz] = 0;
sz = varDataLen(pLeft);
char *str = malloc(sz + 1);
memcpy(str, varDataVal(pLeft), sz);
str[sz] = 0;
int errCode = 0;
regex_t regex;
char msgbuf[256] = {0};
int cflags = REG_EXTENDED;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
regfree(&regex);
free(str);
free(pattern);
return 1;
}
errCode = regexec(&regex, str, 0, NULL, 0);
if (errCode != 0 && errCode != REG_NOMATCH) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
}
int32_t result = (errCode == 0) ? 0 : 1;
regfree(&regex);
free(str);
free(pattern);
return result;
}
int32_t taosArrayCompareString(const void* a, const void* b) {
const char* x = *(const char**)a;
const char* y = *(const char**)b;
return compareLenPrefixedStr(x, y);
}
int32_t compareFindItemInSet(const void *pLeft, const void* pRight) {
return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
}
int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
SPatternCompareInfo pInfo = {'%', '_'}; SPatternCompareInfo pInfo = {'%', '_'};
@ -600,4 +218,4 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
} }
} }
} }
} }

2
source/common/src/tdataformat.c
View File

@ -48,7 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
/** /**
* Duplicate the schema and return a new object * Duplicate the schema and return a new object
*/ */
STSchema *tdDupSchema(STSchema *pSchema) { STSchema *tdDupSchema(const STSchema *pSchema) {
int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema);
STSchema *tSchema = (STSchema *)malloc(tlen); STSchema *tSchema = (STSchema *)malloc(tlen);

77
source/common/src/trow.c
View File

@ -11,4 +11,79 @@
* *
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "trow.h"
/* ------------ Structures ---------- */
struct SRowBatch {
int32_t compress : 1; // if batch row is compressed
int32_t nrows : 31; // number of rows
int32_t tlen; // total length (including `nrows` and `tlen`)
char rows[];
};
struct SRowBuilder {
// TODO
};
struct SRowBatchIter {
int32_t counter; // row counter
SRowBatch *rb; // row batch to iter
SRow nrow; // next row
};
struct SRowBatchBuilder {
// TODO
};
/* ------------ Methods ---------- */
// SRowBuilder
SRowBuilder *rowBuilderCreate() {
SRowBuilder *pRowBuilder = NULL;
// TODO
return pRowBuilder;
}
void rowBuilderDestroy(SRowBuilder *pRowBuilder) {
if (pRowBuilder) {
free(pRowBuilder);
}
}
// SRowBatchIter
SRowBatchIter *rowBatchIterCreate(SRowBatch *pRowBatch) {
SRowBatchIter *pRowBatchIter = (SRowBatchIter *)malloc(sizeof(*pRowBatchIter));
if (pRowBatchIter == NULL) {
return NULL;
}
pRowBatchIter->counter = 0;
pRowBatchIter->rb = pRowBatch;
pRowBatchIter->nrow = pRowBatch->rows;
return pRowBatchIter;
};
void rowBatchIterDestroy(SRowBatchIter *pRowBatchIter) {
if (pRowBatchIter) {
free(pRowBatchIter);
}
}
const SRow rowBatchIterNext(SRowBatchIter *pRowBatchIter) {
SRow r = NULL;
if (pRowBatchIter->counter < pRowBatchIter->rb->nrows) {
r = pRowBatchIter->nrow;
pRowBatchIter->counter += 1;
pRowBatchIter->nrow = (SRow)POINTER_SHIFT(r, rowLen(r));
}
return r;
}
// SRowBatchBuilder
SRowBatchBuilder *rowBatchBuilderCreate();
void rowBatchBuilderDestroy(SRowBatchBuilder *);

2
source/common/src/ttime.c
View File

@ -94,8 +94,6 @@ static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t time
parseLocaltimeWithDst parseLocaltimeWithDst
}; };
int32_t taosGetTimestampSec() { return (int32_t)time(NULL); }
int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) {
/* parse datatime string in with tz */ /* parse datatime string in with tz */
if (strnchr(timestr, 'T', len, false) != NULL) { if (strnchr(timestr, 'T', len, false) != NULL) {

4
source/libs/tkv/CMakeLists.txt
View File

@ -4,4 +4,8 @@ target_include_directories(
tkv tkv
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
tkv
PUBLIC os
) )

69
source/libs/tkv/src/tkv.c
View File

@ -11,4 +11,71 @@
* *
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tkv.h"
struct STkvDb {
// TODO
};
struct STkvOpts {
// TODO
};
struct STkvCache {
// TODO
};
struct STkvReadOpts {
// TODO
};
struct STkvWriteOpts {
// TODO
};
STkvDb *tkvOpen(const STkvOpts *options, const char *path) {
// TODO
return NULL;
}
void tkvClose(STkvDb *db) {
// TODO
}
void tkvPut(STkvDb *db, STkvWriteOpts *pwopts, char *key, size_t keylen, char *val, size_t vallen) {
// TODO
}
char *tkvGet(STkvDb *db, STkvReadOpts *propts, char *key, size_t keylen, size_t *vallen) {
// TODO
return NULL;
}
STkvOpts *tkvOptionsCreate() {
// TODO
return NULL;
}
void tkvOptionsDestroy(STkvOpts *popts) {
// TODO
}
void tkvOptionsSetCache(STkvOpts *popts, STkvCache *pCache) {
// TODO
}
STkvReadOpts *tkvReadOptsCreate() {
// TODO
return NULL;
}
void tkvReadOptsDestroy(STkvReadOpts *propts) {
// TODO
}
STkvWriteOpts *tkvWriteOptsCreate() {
// TODO
return NULL;
}
void tkvWriteOptsDestroy(STkvWriteOpts *pwopts) {
// TODO
}

4
source/os/src/osDir.c
View File

@ -34,7 +34,7 @@
#include <unistd.h> #include <unistd.h>
#include <wordexp.h> #include <wordexp.h>
void taosRemoveDir(char *dirname) { void taosRemoveDir(const char *dirname) {
DIR *dir = opendir(dirname); DIR *dir = opendir(dirname);
if (dir == NULL) return; if (dir == NULL) return;
@ -48,7 +48,7 @@ void taosRemoveDir(char *dirname) {
taosRemoveDir(filename); taosRemoveDir(filename);
} else { } else {
(void)remove(filename); (void)remove(filename);
printf("file:%s is removed", filename); printf("file:%s is removed\n", filename);
} }
} }

2
source/os/src/osTime.c
View File

@ -63,4 +63,6 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) {
return gettimeofday(tv, NULL); return gettimeofday(tv, NULL);
} }
int32_t taosGetTimestampSec() { return (int32_t)time(NULL); }
#endif #endif

1
source/server/CMakeLists.txt
View File

@ -8,4 +8,5 @@ add_executable(taosd ${TAOSD_SRC})
target_link_libraries( target_link_libraries(
taosd taosd
PUBLIC dnode PUBLIC dnode
PUBLIC util
) )

9
source/server/vnode/CMakeLists.txt
View File

@ -3,7 +3,7 @@ add_subdirectory(tq)
add_subdirectory(tsdb) add_subdirectory(tsdb)
aux_source_directory(src VNODE_SRC) aux_source_directory(src VNODE_SRC)
add_library(vnode ${VNODE_SRC}) add_library(vnode STATIC ${VNODE_SRC})
target_include_directories( target_include_directories(
vnode vnode
PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode" PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode"
@ -17,4 +17,9 @@ target_link_libraries(
PUBLIC tsdb PUBLIC tsdb
PUBLIC wal PUBLIC wal
PUBLIC cjson PUBLIC cjson
) )
# test
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})

2
source/server/vnode/inc/vnodeInt.h
View File

@ -79,7 +79,7 @@ typedef struct {
void * cq; // continuous query void * cq; // continuous query
int32_t dbCfgVersion; int32_t dbCfgVersion;
int32_t vgCfgVersion; int32_t vgCfgVersion;
STsdbCfg tsdbCfg; // STsdbCfg tsdbCfg;
#if 0 #if 0
SSyncCfg syncCfg; SSyncCfg syncCfg;
#endif #endif

28
source/server/vnode/inc/vnodeMemAllocator.h
View File

@ -16,12 +16,38 @@
#ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_
#define _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_
#include "amalloc.h" #include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct SVnodeMemAllocator SVnodeMemAllocator;
SVnodeMemAllocator *VMACreate(size_t size /* base size */, size_t ssize /* step size */,
size_t threshold /* threshold size when full*/);
void VMADestroy(SVnodeMemAllocator *pvma);
void VMAReset(SVnodeMemAllocator *pvma);
void * VMAMalloc(SVnodeMemAllocator *pvma, size_t size);
void VMAFree(SVnodeMemAllocator *pvma, void *ptr);
bool VMAIsFull(SVnodeMemAllocator *pvma);
// ------------------ FOR TEST ONLY ------------------
typedef struct SVMANode {
struct SVMANode *prev;
size_t tsize;
size_t used;
char data[];
} SVMANode;
struct SVnodeMemAllocator {
bool full; // if allocator is full
size_t threshold; // threshold;
size_t ssize; // step size to allocate
SVMANode *inuse; // inuse node to allocate
SVMANode node; // basic node to use
};
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

9
source/server/vnode/meta/CMakeLists.txt
View File

@ -1,5 +1,5 @@
aux_source_directory(src META_SRC) aux_source_directory(src META_SRC)
add_library(meta ${META_SRC}) add_library(meta STATIC ${META_SRC})
target_include_directories( target_include_directories(
meta meta
PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
@ -8,4 +8,9 @@ target_include_directories(
target_link_libraries( target_link_libraries(
meta meta
PUBLIC common PUBLIC common
) PUBLIC tkv
)
if(${BUILD_TEST})
add_subdirectory(test)
endif(${BUILD_TEST})

34
source/server/vnode/meta/inc/metaUid.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_META_UID_H_
#define _TD_META_UID_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef uint64_t tb_uid_t;
tb_uid_t metaGenerateUid();
#define IVLD_TB_UID 0
#ifdef __cplusplus
}
#endif
#endif /*_TD_META_UID_H_*/

205
source/server/vnode/meta/src/meta.c
View File

@ -13,24 +13,207 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tkv.h"
#include "thash.h"
#include "tlist.h"
#include "tlockfree.h"
#include "ttypes.h"
#include "meta.h" #include "meta.h"
#include "metaUid.h"
/* -------------------- Structures -------------------- */
typedef struct STable {
tb_uid_t uid;
char * name;
tb_uid_t suid;
SArray * schema;
} STable;
typedef struct STableObj {
bool pin;
uint64_t ref;
SRWLatch latch;
uint64_t offset;
SList * ctbList; // child table list
STable * pTable;
} STableObj;
struct SMeta {
pthread_rwlock_t rwLock;
SHashObj *pTableObjHash; // uid --> STableObj
SList * stbList; // super table list
STkvDb * tbnameDb; // tbname --> uid
STkvDb * tagDb; // uid --> tag
STkvDb * schemaDb;
STkvDb * tagIdx;
size_t totalUsed;
};
static STable * metaTableNew(tb_uid_t uid, const char *name, int32_t sver);
static STableObj *metaTableObjNew();
/* -------------------- Methods -------------------- */
SMeta *metaOpen(SMetaOpts *options) {
SMeta *pMeta = NULL;
char * err = NULL;
pMeta = (SMeta *)calloc(1, sizeof(*pMeta));
if (pMeta == NULL) {
return NULL;
}
pthread_rwlock_init(&(pMeta->rwLock), NULL);
pMeta->pTableObjHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
pMeta->stbList = tdListNew(sizeof(STableObj *));
// Options
STkvOpts *dbOptions = tkvOptionsCreate();
taosMkDir("meta");
// Open tbname DB
pMeta->tbnameDb = tkvOpen(dbOptions, "meta/tbname_uid_db");
// Open tag DB
pMeta->tagDb = tkvOpen(dbOptions, "meta/uid_tag_db");
// Open schema DB
pMeta->schemaDb = tkvOpen(dbOptions, "meta/schema_db");
// Open tag index
pMeta->tagIdx = tkvOpen(dbOptions, "meta/tag_idx_db");
tkvOptionsDestroy(dbOptions);
return pMeta;
}
void metaClose(SMeta *pMeta) {
if (pMeta) {
tkvClose(pMeta->tagIdx);
tkvClose(pMeta->schemaDb);
tkvClose(pMeta->tagDb);
tkvClose(pMeta->tbnameDb);
tdListFree(pMeta->stbList);
taosHashCleanup(pMeta->pTableObjHash);
pthread_rwlock_destroy(&(pMeta->rwLock));
}
}
int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) {
size_t vallen;
STkvReadOpts *ropt;
STableObj * pTableObj = NULL;
STkvWriteOpts *wopt;
// Check if table already exists
ropt = tkvReadOptsCreate();
char *uidStr = tkvGet(pMeta->tbnameDb, ropt, pTableOpts->name, strlen(pTableOpts->name), &vallen);
if (uidStr != NULL) {
// Has duplicate named table
return -1;
}
tkvReadOptsDestroy(ropt);
// Create table obj
pTableObj = metaTableObjNew();
if (pTableObj == NULL) {
// TODO
return -1;
}
// Create table object
pTableObj->pTable = metaTableNew(metaGenerateUid(), pTableOpts->name, schemaVersion(pTableOpts->pSchema));
if (pTableObj->pTable == NULL) {
// TODO
}
pthread_rwlock_rdlock(&pMeta->rwLock);
taosHashPut(pMeta->pTableObjHash, &(pTableObj->pTable->uid), sizeof(tb_uid_t), &pTableObj, sizeof(pTableObj));
wopt = tkvWriteOptsCreate();
// rocksdb_writeoptions_disable_WAL(wopt, 1);
// Add to tbname db
tkvPut(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), (char *)&pTableObj->pTable->uid,
sizeof(tb_uid_t));
// Add to schema db
char id[12];
char buf[256];
void *pBuf = buf;
*(tb_uid_t *)id = pTableObj->pTable->uid;
*(int32_t *)(id + sizeof(tb_uid_t)) = schemaVersion(pTableOpts->pSchema);
int size = tdEncodeSchema(&pBuf, pTableOpts->pSchema);
tkvPut(pMeta->schemaDb, wopt, id, 12, buf, size);
tkvWriteOptsDestroy(wopt);
pthread_rwlock_unlock(&pMeta->rwLock);
int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) {
// TODO
return 0; return 0;
} }
int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) { void metaDestroy(const char *path) { taosRemoveDir(path); }
// TODO
return 0; int metaCommit(SMeta *meta) { return 0; }
void metaTableOptsInit(STableOpts *pTableOpts, int8_t type, const char *name, const STSchema *pSchema) {
pTableOpts->type = type;
pTableOpts->name = strdup(name);
pTableOpts->pSchema = tdDupSchema(pSchema);
} }
int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) { /* -------------------- Static Methods -------------------- */
// TODO
return 0; static STable *metaTableNew(tb_uid_t uid, const char *name, int32_t sver) {
STable *pTable = NULL;
pTable = (STable *)malloc(sizeof(*pTable));
if (pTable == NULL) {
// TODO
return NULL;
}
pTable->schema = taosArrayInit(0, sizeof(int32_t));
if (pTable->schema == NULL) {
// TODO
return NULL;
}
pTable->uid = uid;
pTable->name = strdup(name);
pTable->suid = IVLD_TB_UID;
taosArrayPush(pTable->schema, &sver);
return pTable;
} }
int metaCommit(SMeta *pMeta) { static STableObj *metaTableObjNew() {
// TODO STableObj *pTableObj = NULL;
return 0;
pTableObj = (STableObj *)malloc(sizeof(*pTableObj));
if (pTableObj == NULL) {
return NULL;
}
pTableObj->pin = true;
pTableObj->ref = 1;
taosInitRWLatch(&(pTableObj->latch));
pTableObj->offset = UINT64_MAX;
pTableObj->ctbList = NULL;
pTableObj->pTable = NULL;
return pTableObj;
} }

23
source/server/vnode/meta/src/metaUid.c Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "metaUid.h"
static tb_uid_t nuid = IVLD_TB_UID;
tb_uid_t metaGenerateUid() {
// TODO: need a more complex UID generator
return ++nuid;
}

24
source/server/vnode/meta/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,24 @@
add_executable(metaTest "")
target_sources(metaTest
PRIVATE
"../src/meta.c"
"../src/metaUid.c"
"metaTests.cpp"
)
target_include_directories(metaTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(metaTest
os
util
common
gtest_main
tkv
)
enable_testing()
add_test(
NAME meta_test
COMMAND metaTest
)

37
source/server/vnode/meta/test/metaTests.cpp
View File

@ -0,0 +1,37 @@
#include <gtest/gtest.h>
#include <string.h>
#include <iostream>
#include "meta.h"
TEST(MetaTest, meta_open_test) {
// Open Meta
SMeta *meta = metaOpen(NULL);
std::cout << "Meta is opened!" << std::endl;
// Create tables
STableOpts tbOpts;
char tbname[128];
STSchema * pSchema;
STSchemaBuilder sb;
tdInitTSchemaBuilder(&sb, 0);
for (size_t i = 0; i < 10; i++) {
tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, i, 8);
}
pSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
for (size_t i = 0; i < 1000000; i++) {
sprintf(tbname, "tb%ld", i);
metaTableOptsInit(&tbOpts, 0, tbname, pSchema);
metaCreateTable(meta, &tbOpts);
}
// Close Meta
metaClose(meta);
std::cout << "Meta is closed!" << std::endl;
// Destroy Meta
metaDestroy("meta");
std::cout << "Meta is destroyed!" << std::endl;
}

112
source/server/vnode/src/vnodeMemAllocator.c
View File

@ -11,4 +11,114 @@
* *
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "vnodeMemAllocator.h"
#define VMA_IS_FULL(pvma) \
(((pvma)->inuse != &((pvma)->node)) || ((pvma)->inuse->tsize - (pvma)->inuse->used < (pvma)->threshold))
static SVMANode *VMANodeNew(size_t size);
static void VMANodeFree(SVMANode *node);
SVnodeMemAllocator *VMACreate(size_t size, size_t ssize, size_t threshold) {
SVnodeMemAllocator *pvma = NULL;
if (size < threshold) {
return NULL;
}
pvma = (SVnodeMemAllocator *)malloc(sizeof(*pvma) + size);
if (pvma) {
pvma->full = false;
pvma->threshold = threshold;
pvma->ssize = ssize;
pvma->inuse = &(pvma->node);
pvma->inuse->prev = NULL;
pvma->inuse->tsize = size;
pvma->inuse->used = 0;
}
return pvma;
}
void VMADestroy(SVnodeMemAllocator *pvma) {
if (pvma) {
VMAReset(pvma);
free(pvma);
}
}
void VMAReset(SVnodeMemAllocator *pvma) {
while (pvma->inuse != &(pvma->node)) {
SVMANode *node = pvma->inuse;
pvma->inuse = node->prev;
VMANodeFree(node);
}
pvma->inuse->used = 0;
pvma->full = false;
}
void *VMAMalloc(SVnodeMemAllocator *pvma, size_t size) {
void * ptr = NULL;
size_t tsize = size + sizeof(size_t);
if (pvma->inuse->tsize - pvma->inuse->used < tsize) {
SVMANode *pNode = VMANodeNew(MAX(pvma->ssize, tsize));
if (pNode == NULL) {
return NULL;
}
pNode->prev = pvma->inuse;
pvma->inuse = pNode;
}
ptr = pvma->inuse->data + pvma->inuse->used;
pvma->inuse->used += tsize;
*(size_t *)ptr = size;
ptr = POINTER_SHIFT(ptr, sizeof(size_t));
pvma->full = VMA_IS_FULL(pvma);
return ptr;
}
void VMAFree(SVnodeMemAllocator *pvma, void *ptr) {
if (ptr) {
size_t size = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t));
if (POINTER_SHIFT(ptr, size) == pvma->inuse->data + pvma->inuse->used) {
pvma->inuse->used -= (size + sizeof(size_t));
if ((pvma->inuse->used == 0) && (pvma->inuse != &(pvma->node))) {
SVMANode *node = pvma->inuse;
pvma->inuse = node->prev;
VMANodeFree(node);
}
pvma->full = VMA_IS_FULL(pvma);
}
}
}
bool VMAIsFull(SVnodeMemAllocator *pvma) { return pvma->full; }
static SVMANode *VMANodeNew(size_t size) {
SVMANode *node = NULL;
node = (SVMANode *)malloc(sizeof(*node) + size);
if (node) {
node->prev = NULL;
node->tsize = size;
node->used = 0;
}
return node;
}
static void VMANodeFree(SVMANode *node) {
if (node) {
free(node);
}
}

9
source/server/vnode/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,9 @@
# vnodeMemAllocatorTest
add_executable(VMATest "")
target_sources(VMATest
PRIVATE
"../src/vnodeMemAllocator.c"
"vnodeMemAllocatorTest.cpp"
)
target_include_directories(VMATest PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc")
target_link_libraries(VMATest os gtest_main)

22
source/server/vnode/test/vnodeMemAllocatorTest.cpp Normal file
View File

@ -0,0 +1,22 @@
#include <gtest/gtest.h>
#include <iostream>
#include "vnodeMemAllocator.h"
TEST(VMATest, basic_create_and_destroy_test) {
SVnodeMemAllocator *vma = VMACreate(1024, 512, 64);
EXPECT_TRUE(vma != nullptr);
EXPECT_EQ(vma->full, false);
EXPECT_EQ(vma->ssize, 512);
EXPECT_EQ(vma->threshold, 64);
EXPECT_EQ(vma->inuse->tsize, 1024);
VMADestroy(vma);
vma = VMACreate(1024, 512, 1024);
EXPECT_TRUE(vma != nullptr);
VMADestroy(vma);
vma = VMACreate(1024, 512, 1025);
EXPECT_TRUE(vma == nullptr);
VMADestroy(vma);
}

2
source/server/vnode/tsdb/src/tsdb.c
View File

@ -15,5 +15,5 @@
#include "tsdb.h" #include "tsdb.h"
int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg) { return 0; } int tsdbInsert(STsdb *tsdb, SSubmitReq *pReq, SSubmitRsp *pRsp) { return 0; }
int tsdbCommit(STsdb *pTsdb) { return 0; } int tsdbCommit(STsdb *pTsdb) { return 0; }

2
source/util/CMakeLists.txt
View File

@ -1,5 +1,5 @@
aux_source_directory(src UTIL_SRC) aux_source_directory(src UTIL_SRC)
add_library(util ${UTIL_SRC}) add_library(util STATIC ${UTIL_SRC})
target_include_directories( target_include_directories(
util util
PUBLIC "${CMAKE_SOURCE_DIR}/include/util" PUBLIC "${CMAKE_SOURCE_DIR}/include/util"

398
source/util/src/compare.c Normal file
View File

@ -0,0 +1,398 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _BSD_SOURCE
#define _GNU_SOURCE
#define _XOPEN_SOURCE
#define _DEFAULT_SOURCE
#include "os.h"
#include "types.h"
#include "compare.h"
#include "ulog.h"
#include "thash.h"
#include "regex.h"
int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
}
int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
}
int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
}
int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
}
int32_t compareInt8Val(const void *pLeft, const void *pRight) {
int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
return compareInt8Val(pRight, pLeft);
}
int32_t compareInt16Val(const void *pLeft, const void *pRight) {
int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
return compareInt16Val(pRight, pLeft);
}
int32_t compareInt32Val(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
return compareInt32Val(pRight, pLeft);
}
int32_t compareInt64Val(const void *pLeft, const void *pRight) {
int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
return compareInt64Val(pRight, pLeft);
}
int32_t compareUint32Val(const void *pLeft, const void *pRight) {
uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
return compareUint32Val(pRight, pLeft);
}
int32_t compareUint64Val(const void *pLeft, const void *pRight) {
uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
return compareUint64Val(pRight, pLeft);
}
int32_t compareUint16Val(const void *pLeft, const void *pRight) {
uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
return compareUint16Val(pRight, pLeft);
}
int32_t compareUint8Val(const void* pLeft, const void* pRight) {
uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
if (left > right) return 1;
if (left < right) return -1;
return 0;
}
int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
return compareUint8Val(pRight, pLeft);
}
int32_t compareFloatVal(const void *pLeft, const void *pRight) {
float p1 = GET_FLOAT_VAL(pLeft);
float p2 = GET_FLOAT_VAL(pRight);
if (isnan(p1) && isnan(p2)) {
return 0;
}
if (isnan(p1)) {
return -1;
}
if (isnan(p2)) {
return 1;
}
if (FLT_EQUAL(p1, p2)) {
return 0;
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
return compareFloatVal(pRight, pLeft);
}
int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
double p1 = GET_DOUBLE_VAL(pLeft);
double p2 = GET_DOUBLE_VAL(pRight);
if (isnan(p1) && isnan(p2)) {
return 0;
}
if (isnan(p1)) {
return -1;
}
if (isnan(p2)) {
return 1;
}
if (FLT_EQUAL(p1, p2)) {
return 0;
}
return FLT_GREATER(p1, p2) ? 1: -1;
}
int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
return compareDoubleVal(pRight, pLeft);
}
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
int32_t len1 = varDataLen(pLeft);
int32_t len2 = varDataLen(pRight);
if (len1 != len2) {
return len1 > len2? 1:-1;
} else {
int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1:-1;
}
}
}
int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
return compareLenPrefixedStr(pRight, pLeft);
}
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
int32_t len1 = varDataLen(pLeft);
int32_t len2 = varDataLen(pRight);
if (len1 != len2) {
return len1 > len2? 1:-1;
} else {
int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
if (ret == 0) {
return 0;
} else {
return ret > 0 ? 1 : -1;
}
}
}
int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
return compareLenPrefixedWStr(pRight, pLeft);
}
/*
* Compare two strings
* TSDB_MATCH: Match
* TSDB_NOMATCH: No match
* TSDB_NOWILDCARDMATCH: No match in spite of having * or % wildcards.
* Like matching rules:
* '%': Matches zero or more characters
* '_': Matches one character
*
*/
int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
char c, c1;
int32_t i = 0;
int32_t j = 0;
int32_t o = 0;
int32_t m = 0;
while ((c = patterStr[i++]) != 0) {
if (c == pInfo->matchAll) { /* Match "*" */
while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
if (c == pInfo->matchOne) {
if (j > size || str[j++] == 0) {
// empty string, return not match
return TSDB_PATTERN_NOWILDCARDMATCH;
} else {
++o;
}
}
}
if (c == 0) {
return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
}
char next[3] = {toupper(c), tolower(c), 0};
m = o;
while (1) {
size_t n = strcspn(str + m, next);
str += m + n;
if (str[0] == 0 || (n >= size)) {
break;
}
int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
m = 0;
}
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
++o;
if (j <= size) {
if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; }
if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
continue;
}
}
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
wchar_t c, c1;
wchar_t matchOne = L'_'; // "_"
wchar_t matchAll = L'%'; // "%"
int32_t i = 0;
int32_t j = 0;
while ((c = patterStr[i++]) != 0) {
if (c == matchAll) { /* Match "%" */
while ((c = patterStr[i++]) == matchAll || c == matchOne) {
if (c == matchOne && (j >= size || str[j++] == 0)) {
return TSDB_PATTERN_NOWILDCARDMATCH;
}
}
if (c == 0) {
return TSDB_PATTERN_MATCH;
}
wchar_t accept[3] = {towupper(c), towlower(c), 0};
while (1) {
size_t n = wcscspn(str, accept);
str += n;
if (str[0] == 0 || (n >= size)) {
break;
}
int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
}
return TSDB_PATTERN_NOWILDCARDMATCH;
}
c1 = str[j++];
if (j <= size) {
if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
continue;
}
}
return TSDB_PATTERN_NOMATCH;
}
return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
}
int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) {
return compareStrRegexComp(pLeft, pRight);
}
int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) {
return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
}
int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
size_t sz = varDataLen(pRight);
char *pattern = malloc(sz + 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
pattern[sz] = 0;
sz = varDataLen(pLeft);
char *str = malloc(sz + 1);
memcpy(str, varDataVal(pLeft), sz);
str[sz] = 0;
int errCode = 0;
regex_t regex;
char msgbuf[256] = {0};
int cflags = REG_EXTENDED;
if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
regfree(&regex);
free(str);
free(pattern);
return 1;
}
errCode = regexec(&regex, str, 0, NULL, 0);
if (errCode != 0 && errCode != REG_NOMATCH) {
regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
}
int32_t result = (errCode == 0) ? 0 : 1;
regfree(&regex);
free(str);
free(pattern);
return result;
}
int32_t taosArrayCompareString(const void* a, const void* b) {
const char* x = *(const char**)a;
const char* y = *(const char**)b;
return compareLenPrefixedStr(x, y);
}
int32_t compareFindItemInSet(const void *pLeft, const void* pRight) {
return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
}

0
source/common/src/terror.c → source/util/src/terror.c
View File

3
source/util/src/thashutil.c
View File

@ -15,8 +15,9 @@
#include "os.h" #include "os.h"
#include "thash.h" #include "thash.h"
#include "tcompare.h" #include "compare.h"
#include "tdef.h" #include "tdef.h"
#include "types.h"
#define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r))) #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r)))

2
source/util/src/tskiplist.c
View File

@ -15,7 +15,7 @@
*/ */
#include "tskiplist.h" #include "tskiplist.h"
#include "os.h" #include "os.h"
#include "tcompare.h" #include "compare.h"
#include "ulog.h" #include "ulog.h"
#include "tutil.h" #include "tutil.h"

21
source/util/src/tutil.c
View File

@ -450,24 +450,3 @@ char *taosIpStr(uint32_t ipInt) {
return ipStr; return ipStr;
} }
FORCE_INLINE float taos_align_get_float(const char* pBuf) {
#if __STDC_VERSION__ >= 201112L
static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
#else
assert(sizeof(float) == sizeof(uint32_t));
#endif
float fv = 0;
memcpy(&fv, pBuf, sizeof(fv)); // in ARM, return *((const float*)(pBuf)) may cause problem
return fv;
}
FORCE_INLINE double taos_align_get_double(const char* pBuf) {
#if __STDC_VERSION__ >= 201112L
static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
#else
assert(sizeof(double) == sizeof(uint64_t));
#endif
double dv = 0;
memcpy(&dv, pBuf, sizeof(dv)); // in ARM, return *((const double*)(pBuf)) may cause problem
return dv;
}