xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #11875 from taosdata/vnode_refact_merge 

refactor: vnode
This commit is contained in:
Hongze Cheng 2022-04-26 17:47:39 +08:00 committed by GitHub
parent 901cf55ff2 15127f3a97
commit d5886dc97d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
59 changed files with 2600 additions and 3015 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -40,6 +40,7 @@ taosdalipu/
Target/
*.failed
*.sql
sim
sim/
psim/
pysim/
@ -107,3 +108,4 @@ TAGS
contrib/*
!contrib/CMakeLists.txt
!contrib/test
sql

136
include/common/tmsg.h
View File

@ -286,6 +286,12 @@ typedef struct SSchema {
#define SSCHMEA_BYTES(s) ((s)->bytes)
#define SSCHMEA_NAME(s) ((s)->name)
typedef struct {
int32_t nCols;
int32_t sver;
SSchema* pSchema;
} SSchemaWrapper;
STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
typedef struct {
@ -1515,59 +1521,81 @@ typedef struct {
char* qmsg2; // pAst2:qmsg2:SRetention2 => trigger aggr task2
} SRSmaParam;
int tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam);
int tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam);
typedef struct SVCreateStbReq {
const char* name;
tb_uid_t suid;
int8_t rollup;
SSchemaWrapper schema;
SSchemaWrapper schemaTag;
SRSmaParam pRSmaParam;
} SVCreateStbReq;
int tEncodeSVCreateStbReq(SCoder* pCoder, const SVCreateStbReq* pReq);
int tDecodeSVCreateStbReq(SCoder* pCoder, SVCreateStbReq* pReq);
typedef struct SVDropStbReq {
// data
} SVDropStbReq;
typedef struct SVCreateStbRsp {
int code;
} SVCreateStbRsp;
typedef struct SVCreateTbReq {
char* name;
uint32_t ttl;
uint32_t keep;
union {
uint8_t info;
struct {
uint8_t rollup : 1; // 1 means rollup sma
uint8_t type : 7;
};
};
tb_uid_t uid;
int64_t ctime;
const char* name;
int32_t ttl;
int8_t type;
union {
struct {
tb_uid_t suid;
col_id_t nCols;
col_id_t nBSmaCols;
SSchema* pSchema;
col_id_t nTagCols;
SSchema* pTagSchema;
SRSmaParam* pRSmaParam;
} stbCfg;
const void* pTag;
} ctb;
struct {
tb_uid_t suid;
SKVRow pTag;
} ctbCfg;
struct {
col_id_t nCols;
col_id_t nBSmaCols;
SSchema* pSchema;
SRSmaParam* pRSmaParam;
} ntbCfg;
SSchemaWrapper schema;
} ntb;
};
} SVCreateTbReq, SVUpdateTbReq;
} SVCreateTbReq;
int tEncodeSVCreateTbReq(SCoder* pCoder, const SVCreateTbReq* pReq);
int tDecodeSVCreateTbReq(SCoder* pCoder, SVCreateTbReq* pReq);
typedef struct {
int32_t nReqs;
union {
SVCreateTbReq* pReqs;
SArray* pArray;
};
} SVCreateTbBatchReq;
int tEncodeSVCreateTbBatchReq(SCoder* pCoder, const SVCreateTbBatchReq* pReq);
int tDecodeSVCreateTbBatchReq(SCoder* pCoder, SVCreateTbBatchReq* pReq);
typedef struct {
int32_t code;
} SVCreateTbRsp, SVUpdateTbRsp;
int tEncodeSVCreateTbRsp(SCoder* pCoder, const SVCreateTbRsp* pRsp);
int tDecodeSVCreateTbRsp(SCoder* pCoder, SVCreateTbRsp* pRsp);
int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq);
typedef struct {
int64_t ver; // use a general definition
int32_t nRsps;
union {
SVCreateTbRsp* pRsps;
SArray* pArray;
} SVCreateTbBatchReq;
int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq);
void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq);
typedef struct {
SArray* rspList; // SArray<SVCreateTbRsp>
};
} SVCreateTbBatchRsp;
int tEncodeSVCreateTbBatchRsp(SCoder* pCoder, const SVCreateTbBatchRsp* pRsp);
int tDecodeSVCreateTbBatchRsp(SCoder* pCoder, SVCreateTbBatchRsp* pRsp);
int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
@ -2087,15 +2115,11 @@ int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset);
int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq);
int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq);
typedef struct {
uint32_t nCols;
SSchema* pSchema;
} SSchemaWrapper;
static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper));
if (pSW == NULL) return pSW;
pSW->nCols = pSchemaWrapper->nCols;
pSW->sver = pSchemaWrapper->sver;
pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
if (pSW->pSchema == NULL) {
taosMemoryFree(pSW);
@ -2132,8 +2156,8 @@ static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) {
static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) {
if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1;
if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1;
if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1;
if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1;
if (tEncodeI32v(pEncoder, pSchema->bytes) < 0) return -1;
if (tEncodeI16v(pEncoder, pSchema->colId) < 0) return -1;
if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1;
return 0;
}
@ -2141,15 +2165,16 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch
static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) {
if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1;
if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1;
if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1;
if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1;
if (tDecodeI32v(pDecoder, &pSchema->bytes) < 0) return -1;
if (tDecodeI16v(pDecoder, &pSchema->colId) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1;
return 0;
}
static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) {
int32_t tlen = 0;
tlen += taosEncodeFixedU32(buf, pSW->nCols);
tlen += taosEncodeVariantI32(buf, pSW->nCols);
tlen += taosEncodeVariantI32(buf, pSW->sver);
for (int32_t i = 0; i < pSW->nCols; i++) {
tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]);
}
@ -2157,7 +2182,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr
}
static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) {
buf = taosDecodeFixedU32(buf, &pSW->nCols);
buf = taosDecodeVariantI32(buf, &pSW->nCols);
buf = taosDecodeVariantI32(buf, &pSW->sver);
pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
if (pSW->pSchema == NULL) {
return NULL;
@ -2170,23 +2196,25 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp
}
static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) {
if (tEncodeU32(pEncoder, pSW->nCols) < 0) return -1;
if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1;
if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1;
for (int32_t i = 0; i < pSW->nCols; i++) {
if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1;
}
return pEncoder->pos;
return 0;
}
static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapper* pSW) {
if (tDecodeU32(pDecoder, &pSW->nCols) < 0) return -1;
void* ptr = taosMemoryRealloc(pSW->pSchema, pSW->nCols * sizeof(SSchema));
if (ptr == NULL) {
return -1;
}
pSW->pSchema = (SSchema*)ptr;
if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1;
if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1;
pSW->pSchema = (SSchema*)TCODER_MALLOC(pDecoder, sizeof(SSchema) * pSW->nCols);
if (pSW->pSchema == NULL) return -1;
for (int32_t i = 0; i < pSW->nCols; i++) {
if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1;
}
return 0;
}

4
include/common/trow.h
View File

@ -23,7 +23,6 @@
#include "tbuffer.h"
#include "tdataformat.h"
#include "tdef.h"
#include "tschema.h"
#include "ttypes.h"
#include "tutil.h"
@ -341,7 +340,8 @@ static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx,
return false;
}
static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType,
int8_t bitmapMode) {
switch (bitmapMode) {
case 0:
tdGetBitmapValTypeII(pBitmap, colIdx, pValType);

81
include/common/tschema.h
View File

@ -1,81 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_SCHEMA_H_
#define _TD_COMMON_SCHEMA_H_
#include "os.h"
#include "tarray.h"
#include "ttypes.h"
#ifdef __cplusplus
extern "C" {
#endif
#if 0
typedef struct STColumn {
/// column name
char *cname;
union {
/// for encode purpose
uint64_t info;
struct {
uint64_t sma : 1;
/// column data type
uint64_t type : 7;
/// column id
uint64_t cid : 16;
/// max bytes of the column
uint64_t bytes : 32;
/// reserved
uint64_t reserve : 8;
};
};
/// comment about the column
char *comment;
} STColumn;
typedef struct STSchema {
/// schema version
uint16_t sver;
/// number of columns
uint16_t ncols;
/// sma attributes
struct {
bool sma;
SArray *smaArray;
};
/// column info
STColumn cols[];
} STSchema;
typedef struct {
uint64_t size;
STSchema *pSchema;
} STShemaBuilder;
#define tSchemaBuilderInit(target, capacity) \
{ .size = (capacity), .pSchema = (target) }
void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver);
void tSchemaBuilderSetSMA(bool sma, SArray *smaArray);
int32_t tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment);
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_COMMON_SCHEMA_H_*/

37
include/util/tencode.h
View File

@ -17,7 +17,8 @@
#define _TD_UTIL_ENCODE_H_
#include "tcoding.h"
#include "tfreelist.h"
#include "tlist.h"
// #include "tfreelist.h"
#ifdef __cplusplus
extern "C" {
@ -62,10 +63,14 @@ struct SCoderNode {
CODER_NODE_FIELDS
};
typedef struct SCoderMem {
struct SCoderMem* next;
} SCoderMem;
typedef struct {
td_coder_t type;
td_endian_t endian;
SFreeList fl;
SCoderMem* mList;
CODER_NODE_FIELDS
TD_SLIST(SCoderNode) stack;
} SCoder;
@ -74,7 +79,31 @@ typedef struct {
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
#define TCODER_MALLOC(PTR, TYPE, SIZE, CODER) TFL_MALLOC(PTR, TYPE, SIZE, &((CODER)->fl))
#define TCODER_MALLOC(PCODER, SIZE) \
({ \
void* ptr = NULL; \
SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \
if (pMem) { \
pMem->next = (PCODER)->mList; \
(PCODER)->mList = pMem; \
ptr = (void*)&pMem[1]; \
} \
ptr; \
})
#define tEncodeSize(E, S, SIZE) \
({ \
SCoder coder = {0}; \
int ret = 0; \
tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
if ((E)(&coder, S) == 0) { \
SIZE = coder.pos; \
} else { \
ret = -1; \
} \
tCoderClear(&coder); \
ret; \
})
void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type);
void tCoderClear(SCoder* pCoder);
@ -377,7 +406,9 @@ static FORCE_INLINE int32_t tDecodeBinary(SCoder* pDecoder, const void** val, ui
if (tDecodeU64v(pDecoder, len) < 0) return -1;
if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1;
if (val) {
*val = (void*)TD_CODER_CURRENT(pDecoder);
}
TD_CODER_MOVE_POS(pDecoder, *len);
return 0;

3
include/util/tlog.h
View File

@ -59,6 +59,7 @@ extern int32_t sDebugFlag;
extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag;
extern int32_t fsDebugFlag;
extern int32_t metaDebugFlag;
extern int32_t fnDebugFlag;
int32_t taosInitLog(const char *logName, int32_t maxFiles);
@ -79,6 +80,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
#endif
;
// clang-format off
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
@ -88,6 +90,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
#define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
// clang-format on
#ifdef __cplusplus
}

404
source/common/src/tmsg.c
View File

@ -490,181 +490,6 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR
return 0;
}
int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeFixedU32(buf, pReq->ttl);
tlen += taosEncodeFixedU32(buf, pReq->keep);
tlen += taosEncodeFixedU8(buf, pReq->info);
switch (pReq->type) {
case TD_SUPER_TABLE:
tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nCols);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type);
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].flags);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId);
tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name);
}
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols);
for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type);
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].flags);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId);
tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
}
if (pReq->rollup && pReq->stbCfg.pRSmaParam) {
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
tlen += taosEncodeFixedI32(buf, param->delay);
tlen += taosEncodeFixedI32(buf, param->qmsg1Len);
if (param->qmsg1Len > 0) {
tlen += taosEncodeString(buf, param->qmsg1);
}
tlen += taosEncodeFixedI32(buf, param->qmsg2Len);
if (param->qmsg2Len > 0) {
tlen += taosEncodeString(buf, param->qmsg2);
}
}
break;
case TD_CHILD_TABLE:
tlen += taosEncodeFixedI64(buf, pReq->ctbCfg.suid);
tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag);
break;
case TD_NORMAL_TABLE:
tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nCols);
tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type);
tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].flags);
tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId);
tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
}
if (pReq->rollup && pReq->ntbCfg.pRSmaParam) {
SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
tlen += taosEncodeFixedI32(buf, param->delay);
}
break;
default:
ASSERT(0);
}
return tlen;
}
void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeString(buf, &(pReq->name));
buf = taosDecodeFixedU32(buf, &(pReq->ttl));
buf = taosDecodeFixedU32(buf, &(pReq->keep));
buf = taosDecodeFixedU8(buf, &(pReq->info));
switch (pReq->type) {
case TD_SUPER_TABLE:
buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].flags));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes));
buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name);
}
buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols);
pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].flags));
buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
}
if (pReq->rollup) {
pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam));
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
buf = taosDecodeFixedI32(buf, &param->delay);
buf = taosDecodeFixedI32(buf, &param->qmsg1Len);
if (param->qmsg1Len > 0) {
buf = taosDecodeString(buf, &param->qmsg1);
}
buf = taosDecodeFixedI32(buf, &param->qmsg2Len);
if (param->qmsg2Len > 0) {
buf = taosDecodeString(buf, &param->qmsg2);
}
} else {
pReq->stbCfg.pRSmaParam = NULL;
}
break;
case TD_CHILD_TABLE:
buf = taosDecodeFixedI64(buf, &pReq->ctbCfg.suid);
buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag);
break;
case TD_NORMAL_TABLE:
buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].flags);
buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);
buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
}
if (pReq->rollup) {
pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
buf = taosDecodeFixedI32(buf, &param->delay);
} else {
pReq->ntbCfg.pRSmaParam = NULL;
}
break;
default:
ASSERT(0);
}
return buf;
}
int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->ver);
tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray));
for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) {
SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i);
tlen += tSerializeSVCreateTbReq(buf, pCreateTbReq);
}
return tlen;
}
void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
uint32_t nsize = 0;
buf = taosDecodeFixedI64(buf, &pReq->ver);
buf = taosDecodeFixedU32(buf, &nsize);
pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq));
for (size_t i = 0; i < nsize; i++) {
SVCreateTbReq req = {0};
buf = tDeserializeSVCreateTbReq(buf, &req);
taosArrayPush(pReq->pArray, &req);
}
return buf;
}
int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->ver);
@ -3420,7 +3245,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq
int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) {
if (tStartDecode(decoder) < 0) return -1;
if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
TCODER_MALLOC(pReq->offsets, SMqOffset *, pReq->num * sizeof(SMqOffset), decoder);
pReq->offsets = (SMqOffset *)TCODER_MALLOC(decoder, sizeof(SMqOffset) * pReq->num);
if (pReq->offsets == NULL) return -1;
for (int32_t i = 0; i < pReq->num; i++) {
tDecodeSMqOffset(decoder, &pReq->offsets[i]);
@ -3636,48 +3461,79 @@ int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pR
}
int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
// SCoder encoder = {0};
// tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (pRsp->rspList) {
int32_t num = taosArrayGetSize(pRsp->rspList);
if (tEncodeI32(&encoder, num) < 0) return -1;
for (int32_t i = 0; i < num; ++i) {
SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
}
} else {
if (tEncodeI32(&encoder, 0) < 0) return -1;
}
tEndEncode(&encoder);
// if (tStartEncode(&encoder) < 0) return -1;
// if (pRsp->rspList) {
// int32_t num = taosArrayGetSize(pRsp->rspList);
// if (tEncodeI32(&encoder, num) < 0) return -1;
// for (int32_t i = 0; i < num; ++i) {
// SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
// if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
// }
// } else {
// if (tEncodeI32(&encoder, 0) < 0) return -1;
// }
// tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
// int32_t tlen = encoder.pos;
// tCoderClear(&encoder);
// reture tlen;
return 0;
}
int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
SCoder decoder = {0};
int32_t num = 0;
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
// SCoder decoder = {0};
// int32_t num = 0;
// tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &num) < 0) return -1;
if (num > 0) {
pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp));
if (NULL == pRsp->rspList) return -1;
for (int32_t i = 0; i < num; ++i) {
SVCreateTbRsp rsp = {0};
if (tDecodeI32(&decoder, &rsp.code) < 0) return -1;
if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1;
}
} else {
pRsp->rspList = NULL;
}
tEndDecode(&decoder);
// if (tStartDecode(&decoder) < 0) return -1;
// if (tDecodeI32(&decoder, &num) < 0) return -1;
// if (num > 0) {
// pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp));
// if (NULL == pRsp->rspList) return -1;
// for (int32_t i = 0; i < num; ++i) {
// SVCreateTbRsp rsp = {0};
// if (tDecodeI32(&decoder, &rsp.code) < 0) return -1;
// if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1;
// }
// } else {
// pRsp->rspList = NULL;
// }
// tEndDecode(&decoder);
tCoderClear(&decoder);
// tCoderClear(&decoder);
return 0;
}
int tEncodeSVCreateTbBatchRsp(SCoder *pCoder, const SVCreateTbBatchRsp *pRsp) {
int32_t nRsps = taosArrayGetSize(pRsp->pArray);
SVCreateTbRsp *pCreateRsp;
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI32v(pCoder, nRsps) < 0) return -1;
for (int32_t i = 0; i < nRsps; i++) {
pCreateRsp = taosArrayGet(pRsp->pArray, i);
if (tEncodeSVCreateTbRsp(pCoder, pCreateRsp) < 0) return -1;
}
tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateTbBatchRsp(SCoder *pCoder, SVCreateTbBatchRsp *pRsp) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI32v(pCoder, &pRsp->nRsps) < 0) return -1;
pRsp->pRsps = (SVCreateTbRsp *)TCODER_MALLOC(pCoder, sizeof(*pRsp->pRsps) * pRsp->nRsps);
for (int32_t i = 0; i < pRsp->nRsps; i++) {
if (tDecodeSVCreateTbRsp(pCoder, pRsp->pRsps + i) < 0) return -1;
}
tEndDecode(pCoder);
return 0;
}
@ -3779,6 +3635,38 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
taosMemoryFreeClear(pReq->ast);
}
int tEncodeSVCreateStbReq(SCoder *pCoder, const SVCreateStbReq *pReq) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeCStr(pCoder, pReq->name) < 0) return -1;
if (tEncodeI64(pCoder, pReq->suid) < 0) return -1;
if (tEncodeI8(pCoder, pReq->rollup) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
// if (pReq->rollup) {
// if (tEncodeSRSmaParam(pCoder, pReq->pRSmaParam) < 0) return -1;
// }
tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateStbReq(SCoder *pCoder, SVCreateStbReq *pReq) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1;
if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1;
if (tDecodeI8(pCoder, &pReq->rollup) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
// if (pReq->rollup) {
// if (tDecodeSRSmaParam(pCoder, pReq->pRSmaParam) < 0) return -1;
// }
tEndDecode(pCoder);
return 0;
}
STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
STSchemaBuilder schemaBuilder = {0};
if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) {
@ -3802,3 +3690,97 @@ STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
tdDestroyTSchemaBuilder(&schemaBuilder);
return pNSchema;
}
int tEncodeSVCreateTbReq(SCoder *pCoder, const SVCreateTbReq *pReq) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI64(pCoder, pReq->uid) < 0) return -1;
if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1;
if (tEncodeCStr(pCoder, pReq->name) < 0) return -1;
if (tEncodeI32(pCoder, pReq->ttl) < 0) return -1;
if (tEncodeI8(pCoder, pReq->type) < 0) return -1;
if (pReq->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1;
if (tEncodeBinary(pCoder, pReq->ctb.pTag, kvRowLen(pReq->ctb.pTag)) < 0) return -1;
} else if (pReq->type == TSDB_NORMAL_TABLE) {
if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schema) < 0) return -1;
} else {
ASSERT(0);
}
tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateTbReq(SCoder *pCoder, SVCreateTbReq *pReq) {
uint64_t len;
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1;
if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1;
if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1;
if (tDecodeI32(pCoder, &pReq->ttl) < 0) return -1;
if (tDecodeI8(pCoder, &pReq->type) < 0) return -1;
if (pReq->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1;
if (tDecodeBinary(pCoder, &pReq->ctb.pTag, &len) < 0) return -1;
} else if (pReq->type == TSDB_NORMAL_TABLE) {
if (tDecodeSSchemaWrapper(pCoder, &pReq->ntb.schema) < 0) return -1;
} else {
ASSERT(0);
}
tEndDecode(pCoder);
return 0;
}
int tEncodeSVCreateTbBatchReq(SCoder *pCoder, const SVCreateTbBatchReq *pReq) {
int32_t nReq = taosArrayGetSize(pReq->pArray);
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI32v(pCoder, nReq) < 0) return -1;
for (int iReq = 0; iReq < nReq; iReq++) {
if (tEncodeSVCreateTbReq(pCoder, (SVCreateTbReq *)taosArrayGet(pReq->pArray, iReq)) < 0) return -1;
}
tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateTbBatchReq(SCoder *pCoder, SVCreateTbBatchReq *pReq) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI32v(pCoder, &pReq->nReqs) < 0) return -1;
pReq->pReqs = (SVCreateTbReq *)TCODER_MALLOC(pCoder, sizeof(SVCreateTbReq) * pReq->nReqs);
if (pReq->pReqs == NULL) return -1;
for (int iReq = 0; iReq < pReq->nReqs; iReq++) {
if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1;
}
tEndDecode(pCoder);
return 0;
}
int tEncodeSVCreateTbRsp(SCoder *pCoder, const SVCreateTbRsp *pRsp) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI32(pCoder, pRsp->code) < 0) return -1;
tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateTbRsp(SCoder *pCoder, SVCreateTbRsp *pRsp) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI32(pCoder, &pRsp->code) < 0) return -1;
tEndDecode(pCoder);
return 0;
}

6
source/common/test/tschemaTest.cpp
View File

@ -1,6 +0,0 @@
#include <gtest/gtest.h>
#include "tschema.h"
TEST(td_schema_test, build_schema_test) {
}

12
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -107,14 +107,13 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->vgId = pCreate->vgId;
strcpy(pCfg->dbname, pCreate->db);
pCfg->wsize = pCreate->cacheBlockSize * 1024 * 1024;
pCfg->ssize = 1024;
pCfg->lsize = 1024 * 1024;
pCfg->szBuf = pCreate->cacheBlockSize * 1024 * 1024;
pCfg->streamMode = pCreate->streamMode;
pCfg->isWeak = true;
pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
pCfg->tsdbCfg.keep0 = pCreate->daysToKeep2;
pCfg->tsdbCfg.keep1 = pCreate->daysToKeep0;
pCfg->tsdbCfg.days = 10;
pCfg->tsdbCfg.keep2 = 3650; // pCreate->daysToKeep0;
pCfg->tsdbCfg.keep0 = 3650; // pCreate->daysToKeep2;
pCfg->tsdbCfg.keep1 = 3650; // pCreate->daysToKeep0;
pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
pCfg->tsdbCfg.retentions = pCreate->pRetensions;
pCfg->walCfg.vgId = pCreate->vgId;
@ -212,7 +211,6 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
return code;
}
code = vmWriteVnodesToFile(pMgmt);
if (code != 0) {
tFreeSCreateVnodeReq(&createReq);

88
source/dnode/mnode/impl/src/mndStb.c
View File

@ -73,7 +73,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) +
+ pStb->commentLen + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE;
+pStb->commentLen + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER;
@ -393,72 +393,54 @@ static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSch
}
static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) {
SCoder coder = {0};
int32_t contLen;
SName name = {0};
SVCreateStbReq req = {0};
tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
char dbFName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&name, dbFName);
SVCreateTbReq req = {0};
req.name = (char *)tNameGetTableName(&name);
req.ttl = 0;
req.keep = 0;
req.rollup = pStb->pAst1 > 0 ? 1 : 0;
req.type = TD_SUPER_TABLE;
req.stbCfg.suid = pStb->uid;
req.stbCfg.nCols = pStb->numOfColumns;
req.stbCfg.nTagCols = pStb->numOfTags;
req.stbCfg.pTagSchema = pStb->pTags;
req.stbCfg.nBSmaCols = pStb->numOfSmas;
req.stbCfg.pSchema = (SSchema *)taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema));
if (req.stbCfg.pSchema == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
req.suid = pStb->uid;
req.rollup = pStb->ast1Len > 0 ? 1 : 0;
req.schema.nCols = pStb->numOfColumns;
req.schema.sver = 0;
req.schema.pSchema = pStb->pColumns;
req.schemaTag.nCols = pStb->numOfTags;
req.schemaTag.pSchema = pStb->pTags;
// TODO: remove here
for (int iCol = 0; iCol < req.schema.nCols; iCol++) {
req.schema.pSchema[iCol].flags = SCHEMA_SMA_ON;
}
memcpy(req.stbCfg.pSchema, pStb->pColumns, sizeof(SSchema) * pStb->numOfColumns);
for (int i = 0; i < pStb->numOfColumns; i++) {
req.stbCfg.pSchema[i].flags = SCHEMA_SMA_ON;
}
SRSmaParam *pRSmaParam = NULL;
if (req.rollup) {
pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam));
if (pRSmaParam == NULL) {
taosMemoryFreeClear(req.stbCfg.pSchema);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pRSmaParam->xFilesFactor = pStb->xFilesFactor;
pRSmaParam->delay = pStb->delay;
req.pRSmaParam.xFilesFactor = pStb->xFilesFactor;
req.pRSmaParam.delay = pStb->delay;
if (pStb->ast1Len > 0) {
if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(req.stbCfg.pRSmaParam);
taosMemoryFreeClear(req.stbCfg.pSchema);
if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len) != TSDB_CODE_SUCCESS) {
return NULL;
}
}
if (pStb->ast2Len > 0) {
if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pRSmaParam->qmsg1);
taosMemoryFreeClear(req.stbCfg.pRSmaParam);
taosMemoryFreeClear(req.stbCfg.pSchema);
if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len) != TSDB_CODE_SUCCESS) {
return NULL;
}
}
req.stbCfg.pRSmaParam = pRSmaParam;
}
// get length
if (tEncodeSize(tEncodeSVCreateStbReq, &req, contLen) < 0) {
return NULL;
}
int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
contLen += sizeof(SMsgHead);
SMsgHead *pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) {
if (pRSmaParam) {
taosMemoryFreeClear(pRSmaParam->qmsg1);
taosMemoryFreeClear(pRSmaParam->qmsg2);
taosMemoryFreeClear(pRSmaParam);
}
taosMemoryFreeClear(req.stbCfg.pSchema);
taosMemoryFreeClear(req.pRSmaParam.qmsg1);
taosMemoryFreeClear(req.pRSmaParam.qmsg2);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -467,15 +449,15 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
pHead->vgId = htonl(pVgroup->vgId);
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tSerializeSVCreateTbReq(&pBuf, &req);
tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, contLen - sizeof(SMsgHead), TD_ENCODER);
if (tEncodeSVCreateStbReq(&coder, &req) < 0) {
return NULL;
}
tCoderClear(&coder);
*pContLen = contLen;
if (pRSmaParam) {
taosMemoryFreeClear(pRSmaParam->qmsg1);
taosMemoryFreeClear(pRSmaParam->qmsg2);
taosMemoryFreeClear(pRSmaParam);
}
taosMemoryFreeClear(req.stbCfg.pSchema);
taosMemoryFreeClear(req.pRSmaParam.qmsg1);
taosMemoryFreeClear(req.pRSmaParam.qmsg2);
return pHead;
}

10
source/dnode/vnode/CMakeLists.txt
View File

@ -5,9 +5,7 @@ target_sources(
PRIVATE
# vnode
"src/vnd/vnodeOpen.c"
"src/vnd/vnodeArenaMAImpl.c"
"src/vnd/vnodeBufferPool.c"
# "src/vnd/vnodeBufferPool2.c"
"src/vnd/vnodeBufPool.c"
"src/vnd/vnodeCfg.c"
"src/vnd/vnodeCommit.c"
"src/vnd/vnodeInt.c"
@ -22,11 +20,14 @@ target_sources(
"src/meta/metaIdx.c"
"src/meta/metaTable.c"
"src/meta/metaTDBImpl.c"
# "src/meta/metaBDBImpl.c"
"src/meta/metaQuery.c"
"src/meta/metaCommit.c"
"src/meta/metaEntry.c"
# tsdb
"src/tsdb/tsdbTDBImpl.c"
"src/tsdb/tsdbCommit.c"
"src/tsdb/tsdbCommit2.c"
"src/tsdb/tsdbCompact.c"
"src/tsdb/tsdbFile.c"
"src/tsdb/tsdbFS.c"
@ -71,6 +72,7 @@ target_link_libraries(
PUBLIC transport
PUBLIC stream
)
target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
if(${BUILD_TEST})
add_subdirectory(test)

68
source/dnode/vnode/inc/vnode.h
View File

@ -32,6 +32,8 @@
#include "tmsg.h"
#include "trow.h"
#include "tdbInt.h"
#ifdef __cplusplus
extern "C" {
#endif
@ -59,7 +61,7 @@ int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg);
int32_t vnodeCompact(SVnode *pVnode);
int32_t vnodeSync(SVnode *pVnode);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName);
int vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
int32_t vnodeStart(SVnode *pVnode);
void vnodeStop(SVnode *pVnode);
@ -69,14 +71,23 @@ void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
// meta
typedef struct SMeta SMeta; // todo: remove
typedef struct SMTbCursor SMTbCursor;
typedef struct SMetaReader SMetaReader;
typedef struct SMetaEntry SMetaEntry;
void metaReaderInit(SMetaReader *pReader, SVnode *pVnode, int32_t flags);
void metaReaderClear(SMetaReader *pReader);
int metaReadNext(SMetaReader *pReader);
#if 1 // refact APIs below (TODO)
typedef SVCreateTbReq STbCfg;
typedef SVCreateTSmaReq SSmaCfg;
typedef struct SMTbCursor SMTbCursor;
SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
void metaCloseTbCursor(SMTbCursor *pTbCur);
char *metaTbCursorNext(SMTbCursor *pTbCur);
int metaTbCursorNext(SMTbCursor *pTbCur);
#endif
// tsdb
typedef struct STsdb STsdb;
@ -130,6 +141,7 @@ struct STsdbCfg {
int8_t precision;
int8_t update;
int8_t compression;
int8_t slLevel;
int32_t days;
int32_t minRows;
int32_t maxRows;
@ -144,10 +156,10 @@ struct SVnodeCfg {
int32_t vgId;
char dbname[TSDB_DB_NAME_LEN];
uint64_t dbId;
uint64_t wsize;
uint64_t ssize;
uint64_t lsize;
bool isHeapAllocator;
int32_t szPage;
int32_t szCache;
uint64_t szBuf;
bool isHeap;
uint32_t ttl;
uint32_t keep;
int8_t streamMode;
@ -165,6 +177,48 @@ typedef struct {
uint64_t uid;
} STableKeyInfo;
struct SMetaEntry {
int64_t version;
int8_t type;
tb_uid_t uid;
const char *name;
union {
struct {
SSchemaWrapper schema;
SSchemaWrapper schemaTag;
} stbEntry;
struct {
int64_t ctime;
int32_t ttlDays;
tb_uid_t suid;
const void *pTags;
} ctbEntry;
struct {
int64_t ctime;
int32_t ttlDays;
SSchemaWrapper schema;
} ntbEntry;
};
};
struct SMetaReader {
int32_t flags;
SMeta *pMeta;
SCoder coder;
SMetaEntry me;
void *pBuf;
int szBuf;
};
struct SMTbCursor {
TDBC *pDbc;
void *pKey;
void *pVal;
int kLen;
int vLen;
SMetaReader mr;
};
#ifdef __cplusplus
}
#endif

76
source/dnode/vnode/src/inc/meta.h
View File

@ -39,25 +39,82 @@ typedef struct SMSmaCursor SMSmaCursor;
int metaOpen(SVnode* pVnode, SMeta** ppMeta);
int metaClose(SMeta* pMeta);
// metaEntry ==================
int metaEncodeEntry(SCoder* pCoder, const SMetaEntry* pME);
int metaDecodeEntry(SCoder* pCoder, SMetaEntry* pME);
// metaTable ==================
int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
// metaQuery ==================
int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid);
int metaGetTableEntryByUid(SMetaReader* pReader, tb_uid_t uid);
int metaGetTableEntryByName(SMetaReader* pReader, const char* name);
// metaIdx ==================
int metaOpenIdx(SMeta* pMeta);
void metaCloseIdx(SMeta* pMeta);
int metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions);
int metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
// metaCommit ==================
int metaBegin(SMeta* pMeta);
static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }
struct SMeta {
char* path;
SVnode* pVnode;
TENV* pEnv;
TXN txn;
TDB* pTbDb;
TDB* pSkmDb;
TDB* pUidIdx;
TDB* pNameIdx;
TDB* pCtbIdx;
TDB* pTagIdx;
TDB* pTtlIdx;
SMetaIdx* pIdx;
};
typedef struct {
int64_t version;
tb_uid_t uid;
} STbDbKey;
typedef struct __attribute__((__packed__)) {
tb_uid_t uid;
int32_t sver;
} SSkmDbKey;
typedef struct {
tb_uid_t suid;
tb_uid_t uid;
} SCtbIdxKey;
typedef struct __attribute__((__packed__)) {
tb_uid_t suid;
int16_t cid;
char data[];
} STagIdxKey;
typedef struct {
int64_t dtime;
tb_uid_t uid;
} STtlIdxKey;
#if 1
#define META_SUPER_TABLE TD_SUPER_TABLE
#define META_CHILD_TABLE TD_CHILD_TABLE
#define META_NORMAL_TABLE TD_NORMAL_TABLE
int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
// int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaDropTable(SMeta* pMeta, tb_uid_t uid);
int metaCommit(SMeta* pMeta);
int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
STbCfg* metaGetTbInfoByUid(SMeta* pMeta, tb_uid_t uid);
STbCfg* metaGetTbInfoByName(SMeta* pMeta, char* tbname, tb_uid_t* uid);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
@ -71,6 +128,7 @@ SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
void metaCloseCtbCurosr(SMCtbCursor* pCtbCur);
tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
#ifndef META_REFACT
// SMetaDB
int metaOpenDB(SMeta* pMeta);
void metaCloseDB(SMeta* pMeta);
@ -78,17 +136,9 @@ int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
#endif
// SMetaIdx
tb_uid_t metaGenerateUid(SMeta* pMeta);
struct SMeta {
char* path;
SVnode* pVnode;
SMetaDB* pDB;
SMetaIdx* pIdx;
};
#endif
#ifdef __cplusplus
}

61
source/dnode/vnode/src/inc/tsdb.h
View File

@ -30,22 +30,39 @@ extern "C" {
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on
// tsdbMemTable ================
typedef struct STbData STbData;
typedef struct STsdbMemTable STsdbMemTable;
typedef struct SMergeInfo SMergeInfo;
typedef struct STable STable;
int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable);
void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable);
int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows);
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
// tsdbCommit ================
int tsdbBegin(STsdb *pTsdb);
#if 1
typedef struct SSmaStat SSmaStat;
typedef struct SSmaEnv SSmaEnv;
typedef struct SSmaEnvs SSmaEnvs;
typedef struct STable {
struct STable {
uint64_t tid;
uint64_t uid;
STSchema *pSchema;
} STable;
};
#define TABLE_TID(t) (t)->tid
#define TABLE_UID(t) (t)->uid
STsdb *tsdbOpen(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF);
void tsdbClose(STsdb *);
int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb);
int tsdbClose(STsdb *pTsdb);
int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp);
int tsdbPrepareCommit(STsdb *pTsdb);
int tsdbCommit(STsdb *pTsdb);
int32_t tsdbInitSma(STsdb *pTsdb);
@ -108,25 +125,28 @@ typedef struct {
TSKEY minKey;
} SRtn;
typedef struct STbData {
struct STbData {
tb_uid_t uid;
TSKEY keyMin;
TSKEY keyMax;
int64_t minVer;
int64_t maxVer;
int64_t nrows;
SSkipList *pData;
} STbData;
};
typedef struct STsdbMemTable {
struct STsdbMemTable {
SVBufPool *pPool;
T_REF_DECLARE()
SRWLatch latch;
TSKEY keyMin;
TSKEY keyMax;
uint64_t nRow;
SMemAllocator *pMA;
// Container
int64_t minVer;
int64_t maxVer;
int64_t nRow;
SSkipList *pSlIdx; // SSkiplist<STbData>
SHashObj *pHashIdx;
} STsdbMemTable;
};
typedef struct {
uint32_t version; // Commit version from 0 to increase
@ -152,16 +172,15 @@ typedef struct {
} STsdbFS;
struct STsdb {
int32_t vgId;
char *path;
SVnode *pVnode;
int32_t vgId;
bool repoLocked;
TdThreadMutex mutex;
char *path;
STsdbCfg config;
STsdbMemTable *mem;
STsdbMemTable *imem;
SRtn rtn;
SMemAllocatorFactory *pmaf;
STsdbFS *fs;
SSmaEnvs smaEnvs;
};
@ -185,7 +204,7 @@ static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock,
}
// tsdbMemTable.h
typedef struct {
struct SMergeInfo {
int rowsInserted;
int rowsUpdated;
int rowsDeleteSucceed;
@ -193,7 +212,7 @@ typedef struct {
int nOperations;
TSKEY keyFirst;
TSKEY keyLast;
} SMergeInfo;
};
static void *taosTMalloc(size_t size);
static void *taosTCalloc(size_t nmemb, size_t size);
@ -202,12 +221,6 @@ static void *taosTZfree(void *ptr);
static size_t taosTSizeof(void *ptr);
static void taosTMemset(void *ptr, int c);
STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb);
void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable);
int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp);
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) {
if (pIter == NULL) return NULL;
@ -987,6 +1000,8 @@ static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void
return len;
}
#endif
#ifdef __cplusplus
}
#endif

76
source/dnode/vnode/src/inc/vnd.h
View File

@ -31,70 +31,54 @@ extern "C" {
// clang-format on
// vnodeCfg ====================
extern const SVnodeCfg vnodeCfgDefault;
int vnodeCheckCfg(const SVnodeCfg*);
int vnodeEncodeConfig(const void* pObj, SJson* pJson);
int vnodeDecodeConfig(const SJson* pJson, void* pObj);
// vnodeModule ====================
int vnodeScheduleTask(int (*execute)(void*), void* arg);
// vnodeBufPool ====================
typedef struct SVBufPoolNode SVBufPoolNode;
struct SVBufPoolNode {
SVBufPoolNode* prev;
SVBufPoolNode** pnext;
int64_t size;
uint8_t data[];
};
struct SVBufPool {
SVBufPool* next;
int64_t nRef;
int64_t size;
uint8_t* ptr;
SVBufPoolNode* pTail;
SVBufPoolNode node;
};
int vnodeOpenBufPool(SVnode* pVnode, int64_t size);
int vnodeCloseBufPool(SVnode* pVnode);
void vnodeBufPoolReset(SVBufPool* pPool);
void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
void vnodeBufPoolFree(SVBufPool* pPool, void* p);
// vnodeQuery ====================
int vnodeQueryOpen(SVnode* pVnode);
void vnodeQueryClose(SVnode* pVnode);
int vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg);
// vnodeCommit ====================
int vnodeBegin(SVnode* pVnode);
int vnodeShouldCommit(SVnode* pVnode);
int vnodeCommit(SVnode* pVnode);
int vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg);
int vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo);
int vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo);
int vnodeBegin(SVnode* pVnode, int option);
int vnodeSyncCommit(SVnode* pVnode);
int vnodeAsyncCommit(SVnode* pVnode);
#define vnodeShouldCommit vnodeBufPoolIsFull
#if 1
// SVBufPool
int vnodeOpenBufPool(SVnode* pVnode);
void vnodeCloseBufPool(SVnode* pVnode);
int vnodeBufPoolSwitch(SVnode* pVnode);
int vnodeBufPoolRecycle(SVnode* pVnode);
void* vnodeMalloc(SVnode* pVnode, uint64_t size);
bool vnodeBufPoolIsFull(SVnode* pVnode);
SMemAllocatorFactory* vBufPoolGetMAF(SVnode* pVnode);
// SVMemAllocator
typedef struct SVArenaNode {
TD_SLIST_NODE(SVArenaNode);
uint64_t size; // current node size
void* ptr;
char data[];
} SVArenaNode;
typedef struct SVMemAllocator {
T_REF_DECLARE()
TD_DLIST_NODE(SVMemAllocator);
uint64_t capacity;
uint64_t ssize;
uint64_t lsize;
SVArenaNode* pNode;
TD_SLIST(SVArenaNode) nlist;
} SVMemAllocator;
SVMemAllocator* vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize);
void vmaDestroy(SVMemAllocator* pVMA);
void vmaReset(SVMemAllocator* pVMA);
void* vmaMalloc(SVMemAllocator* pVMA, uint64_t size);
void vmaFree(SVMemAllocator* pVMA, void* ptr);
bool vmaIsFull(SVMemAllocator* pVMA);
// vnodeCfg.h
extern const SVnodeCfg vnodeCfgDefault;
int vnodeCheckCfg(const SVnodeCfg*);
#endif
#ifdef __cplusplus
}
#endif

31
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -22,9 +22,11 @@
#include "sync.h"
#include "tchecksum.h"
#include "tcoding.h"
#include "tcompare.h"
#include "tcompression.h"
#include "tdatablock.h"
#include "tdbInt.h"
#include "tencode.h"
#include "tfs.h"
#include "tglobal.h"
#include "tjson.h"
@ -68,8 +70,8 @@ typedef struct {
} SStreamSinkInfo;
typedef struct {
SVnode *pVnode;
SHashObj *pHash; // streamId -> SStreamSinkInfo
SVnode* pVnode;
SHashObj* pHash; // streamId -> SStreamSinkInfo
} SSink;
// SVState
@ -85,20 +87,23 @@ struct SVnodeInfo {
};
struct SVnode {
char *path;
char* path;
SVnodeCfg config;
SVState state;
STfs *pTfs;
STfs* pTfs;
SMsgCb msgCb;
SVBufPool *pBufPool;
SMeta *pMeta;
STsdb *pTsdb;
SWal *pWal;
STQ *pTq;
SSink *pSink;
int64_t sync; // sync integration
SVBufPool* pPool;
SVBufPool* inUse;
SVBufPool* onCommit;
SVBufPool* onRecycle;
SMeta* pMeta;
STsdb* pTsdb;
SWal* pWal;
STQ* pTq;
SSink* pSink;
int64_t sync;
tsem_t canCommit;
SQHandle *pQuery;
SQHandle* pQuery;
};
#define TD_VID(PVNODE) (PVNODE)->config.vgId
@ -106,7 +111,7 @@ struct SVnode {
typedef struct STbDdlH STbDdlH;
// sma
void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data);
void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
#include "vnd.h"

1048
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

File diff suppressed because it is too large Load Diff

31
source/dnode/vnode/src/meta/metaCommit.c Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBufPoolMalloc((SVBufPool *)pPool, size); }
static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); }
int metaBegin(SMeta *pMeta) {
tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
if (tdbBegin(pMeta->pEnv, &pMeta->txn) < 0) {
return -1;
}
return 0;
}
int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, &pMeta->txn); }

73
source/dnode/vnode/src/meta/metaEntry.c Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
int metaEncodeEntry(SCoder *pCoder, const SMetaEntry *pME) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI64(pCoder, pME->version) < 0) return -1;
if (tEncodeI8(pCoder, pME->type) < 0) return -1;
if (tEncodeI64(pCoder, pME->uid) < 0) return -1;
if (tEncodeCStr(pCoder, pME->name) < 0) return -1;
if (pME->type == TSDB_SUPER_TABLE) {
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
if (tEncodeBinary(pCoder, pME->ctbEntry.pTags, kvRowLen(pME->ctbEntry.pTags)) < 0) return -1;
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
} else {
ASSERT(0);
}
tEndEncode(pCoder);
return 0;
}
int metaDecodeEntry(SCoder *pCoder, SMetaEntry *pME) {
uint64_t len;
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI64(pCoder, &pME->version) < 0) return -1;
if (tDecodeI8(pCoder, &pME->type) < 0) return -1;
if (tDecodeI64(pCoder, &pME->uid) < 0) return -1;
if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
if (pME->type == TSDB_SUPER_TABLE) {
if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
if (tDecodeBinary(pCoder, &pME->ctbEntry.pTags, &len) < 0) return -1; // (TODO)
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
} else {
ASSERT(0);
}
tEndDecode(pCoder);
return 0;
}

16
source/dnode/vnode/src/meta/metaIdx.c
View File

@ -54,7 +54,9 @@ int metaOpenIdx(SMeta *pMeta) {
#ifdef USE_INVERTED_INDEX
SIndexOpts opts;
if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { return -1; }
if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) {
return -1;
}
#endif
return 0;
@ -70,7 +72,9 @@ void metaCloseIdx(SMeta *pMeta) { /* TODO */
#ifdef USE_INVERTED_INDEX
SIndexOpts opts;
if (indexClose(pMeta->pIdx->pIdx) != 0) { return -1; }
if (indexClose(pMeta->pIdx->pIdx) != 0) {
return -1;
}
#endif
}
@ -87,7 +91,7 @@ int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) {
tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id
tb_uid_t tuid = 0; // child table uid
SIndexMultiTerm *terms = indexMultiTermCreate();
SIndexTerm * term =
SIndexTerm *term =
indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid);
indexMultiTermAdd(terms, term);
@ -117,10 +121,13 @@ int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) {
// TODO: add atomicity
#ifdef META_REFACT
#else
if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) {
// TODO: handle error
return -1;
}
#endif
return TSDB_CODE_SUCCESS;
}
@ -128,9 +135,12 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
// TODO: Validate the cfg
// TODO: add atomicity
#ifdef META_REFACT
#else
if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) {
// TODO: handle error
return -1;
}
#endif
return TSDB_CODE_SUCCESS;
}

229
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -15,8 +15,16 @@
#include "vnodeInt.h"
static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
SMeta *pMeta = NULL;
int ret;
int slen;
*ppMeta = NULL;
@ -36,31 +44,240 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
// create path if not created yet
taosMkDir(pMeta->path);
// open meta
if (metaOpenDB(pMeta) < 0) {
// open env
ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv);
if (ret < 0) {
metaError("vgId: %d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
if (metaOpenIdx(pMeta) < 0) {
// open pTbDb
ret = tdbDbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb);
if (ret < 0) {
metaError("vgId: %d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pSkmDb
ret = tdbDbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb);
if (ret < 0) {
metaError("vgId: %d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pUidIdx
ret = tdbDbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx);
if (ret < 0) {
metaError("vgId: %d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pNameIdx
ret = tdbDbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx);
if (ret < 0) {
metaError("vgId: %d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pCtbIdx
ret = tdbDbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx);
if (ret < 0) {
metaError("vgId: %d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pTagIdx
ret = tdbDbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx);
if (ret < 0) {
metaError("vgId: %d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pTtlIdx
ret = tdbDbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx);
if (ret < 0) {
metaError("vgId: %d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open index
if (metaOpenIdx(pMeta) < 0) {
metaError("vgId: %d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
metaDebug("vgId: %d meta is opened", TD_VID(pVnode));
*ppMeta = pMeta;
return 0;
_err:
if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pDB) metaCloseDB(pMeta);
if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
taosMemoryFree(pMeta);
return -1;
}
int metaClose(SMeta *pMeta) {
if (pMeta) {
metaCloseIdx(pMeta);
metaCloseDB(pMeta);
if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
taosMemoryFree(pMeta);
}
return 0;
}
static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
STbDbKey *pTbDbKey1 = (STbDbKey *)pKey1;
STbDbKey *pTbDbKey2 = (STbDbKey *)pKey2;
if (pTbDbKey1->version > pTbDbKey2->version) {
return 1;
} else if (pTbDbKey1->version < pTbDbKey2->version) {
return -1;
}
if (pTbDbKey1->uid > pTbDbKey2->uid) {
return 1;
} else if (pTbDbKey1->uid < pTbDbKey2->uid) {
return -1;
}
return 0;
}
static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
SSkmDbKey *pSkmDbKey1 = (SSkmDbKey *)pKey1;
SSkmDbKey *pSkmDbKey2 = (SSkmDbKey *)pKey2;
if (pSkmDbKey1->uid > pSkmDbKey2->uid) {
return 1;
} else if (pSkmDbKey1->uid < pSkmDbKey2->uid) {
return -1;
}
if (pSkmDbKey1->sver > pSkmDbKey2->sver) {
return 1;
} else if (pSkmDbKey1->sver < pSkmDbKey2->sver) {
return -1;
}
return 0;
}
static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
tb_uid_t uid1 = *(tb_uid_t *)pKey1;
tb_uid_t uid2 = *(tb_uid_t *)pKey2;
if (uid1 > uid2) {
return 1;
} else if (uid1 < uid2) {
return -1;
}
return 0;
}
static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
SCtbIdxKey *pCtbIdxKey1 = (SCtbIdxKey *)pKey1;
SCtbIdxKey *pCtbIdxKey2 = (SCtbIdxKey *)pKey2;
if (pCtbIdxKey1->suid > pCtbIdxKey2->suid) {
return 1;
} else if (pCtbIdxKey1->suid < pCtbIdxKey2->suid) {
return -1;
}
if (pCtbIdxKey1->uid > pCtbIdxKey2->uid) {
return 1;
} else if (pCtbIdxKey1->uid < pCtbIdxKey2->uid) {
return -1;
}
return 0;
}
static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
STagIdxKey *pTagIdxKey1 = (STagIdxKey *)pKey1;
STagIdxKey *pTagIdxKey2 = (STagIdxKey *)pKey2;
int8_t *p1, *p2;
int8_t type;
int c;
// compare suid
if (pTagIdxKey1->suid > pTagIdxKey2->suid) {
return 1;
} else if (pTagIdxKey1->suid < pTagIdxKey2->suid) {
return -1;
}
// compare column id
if (pTagIdxKey1->cid > pTagIdxKey2->cid) {
return 1;
} else if (pTagIdxKey1->cid < pTagIdxKey2->cid) {
return -1;
}
// compare value
p1 = pTagIdxKey1->data;
p2 = pTagIdxKey2->data;
ASSERT(p1[0] == p2[0]);
type = p1[0];
p1++;
p2++;
c = doCompare(p1, p2, type, 0);
if (c) return c;
if (IS_VAR_DATA_TYPE(type)) {
p1 = p1 + varDataTLen(p1);
p2 = p2 + varDataTLen(p2);
} else {
p1 = p1 + tDataTypes[type].bytes;
p2 = p2 + tDataTypes[type].bytes;
}
// compare suid
if (*(tb_uid_t *)p1 > *(tb_uid_t *)p2) {
return 1;
} else if (*(tb_uid_t *)p1 < *(tb_uid_t *)p2) {
return -1;
}
return 0;
}
static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1;
STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2;
if (pTtlIdxKey1->dtime > pTtlIdxKey2->dtime) {
return 1;
} else if (pTtlIdxKey1->dtime < pTtlIdxKey2->dtime) {
return -1;
}
if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {
return 1;
} else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {
return -1;
}
return 0;
}

436
source/dnode/vnode/src/meta/metaQuery.c Normal file
View File

@ -0,0 +1,436 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
void metaReaderInit(SMetaReader *pReader, SVnode *pVnode, int32_t flags) {
memset(pReader, 0, sizeof(*pReader));
pReader->flags = flags;
pReader->pMeta = pVnode->pMeta;
}
void metaReaderClear(SMetaReader *pReader) {
tCoderClear(&pReader->coder);
TDB_FREE(pReader->pBuf);
}
int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t uid) {
SMeta *pMeta = pReader->pMeta;
STbDbKey tbDbKey = {.version = version, .uid = uid};
// query table.db
if (tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) {
goto _err;
}
// decode the entry
tCoderInit(&pReader->coder, TD_LITTLE_ENDIAN, pReader->pBuf, pReader->szBuf, TD_DECODER);
if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) {
goto _err;
}
return 0;
_err:
return -1;
}
int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
SMeta *pMeta = pReader->pMeta;
int64_t version;
// query uid.idx
if (tdbDbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) {
return -1;
}
version = *(int64_t *)pReader->pBuf;
return metaGetTableEntryByVersion(pReader, version, uid);
}
int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
SMeta *pMeta = pReader->pMeta;
tb_uid_t uid;
// query name.idx
if (tdbDbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
return -1;
}
uid = *(tb_uid_t *)pReader->pBuf;
return metaGetTableEntryByUid(pReader, uid);
}
int metaReadNext(SMetaReader *pReader) {
SMeta *pMeta = pReader->pMeta;
// TODO
return 0;
}
#if 1 // ===================================================
SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
SMTbCursor *pTbCur = NULL;
pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur));
if (pTbCur == NULL) {
return NULL;
}
metaReaderInit(&pTbCur->mr, pMeta->pVnode, 0);
tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc);
return pTbCur;
}
void metaCloseTbCursor(SMTbCursor *pTbCur) {
if (pTbCur) {
TDB_FREE(pTbCur->pKey);
TDB_FREE(pTbCur->pVal);
metaReaderClear(&pTbCur->mr);
if (pTbCur->pDbc) {
tdbDbcClose(pTbCur->pDbc);
}
taosMemoryFree(pTbCur);
}
}
int metaTbCursorNext(SMTbCursor *pTbCur) {
int ret;
void *pBuf;
STbCfg tbCfg;
for (;;) {
ret = tdbDbNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
if (ret < 0) {
return -1;
}
metaGetTableEntryByVersion(&pTbCur->mr, *(int64_t *)pTbCur->pVal, *(tb_uid_t *)pTbCur->pKey);
if (pTbCur->mr.me.type == META_SUPER_TABLE) {
continue;
}
break;
}
return 0;
}
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
void *pKey = NULL;
void *pVal = NULL;
int kLen = 0;
int vLen = 0;
int ret;
SSkmDbKey skmDbKey;
SSchemaWrapper *pSW = NULL;
SSchema *pSchema = NULL;
void *pBuf;
SCoder coder = {0};
// fetch
skmDbKey.uid = uid;
skmDbKey.sver = sver;
pKey = &skmDbKey;
kLen = sizeof(skmDbKey);
ret = tdbDbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen);
if (ret < 0) {
return NULL;
}
// decode
pBuf = pVal;
pSW = taosMemoryMalloc(sizeof(pSW));
tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_DECODER);
tDecodeSSchemaWrapper(&coder, pSW);
pSchema = taosMemoryMalloc(sizeof(SSchema) * pSW->nCols);
memcpy(pSchema, pSW->pSchema, sizeof(SSchema) * pSW->nCols);
tCoderClear(&coder);
pSW->pSchema = pSchema;
TDB_FREE(pVal);
return pSW;
}
struct SMCtbCursor {
TDBC *pCur;
tb_uid_t suid;
void *pKey;
void *pVal;
int kLen;
int vLen;
};
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
SMCtbCursor *pCtbCur = NULL;
int ret;
pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur));
if (pCtbCur == NULL) {
return NULL;
}
pCtbCur->suid = uid;
ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur);
if (ret < 0) {
taosMemoryFree(pCtbCur);
return NULL;
}
return pCtbCur;
}
void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
if (pCtbCur) {
if (pCtbCur->pCur) {
tdbDbcClose(pCtbCur->pCur);
TDB_FREE(pCtbCur->pKey);
TDB_FREE(pCtbCur->pVal);
}
taosMemoryFree(pCtbCur);
}
}
tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
int ret;
SCtbIdxKey *pCtbIdxKey;
ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen);
if (ret < 0) {
return 0;
}
pCtbIdxKey = pCtbCur->pKey;
return pCtbIdxKey->uid;
}
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
tb_uid_t quid;
SMetaReader mr = {0};
STSchema *pTSchema = NULL;
SSchemaWrapper *pSW = NULL;
STSchemaBuilder sb = {0};
SSchema *pSchema;
metaReaderInit(&mr, pMeta->pVnode, 0);
metaGetTableEntryByUid(&mr, uid);
if (mr.me.type == TSDB_CHILD_TABLE) {
quid = mr.me.ctbEntry.suid;
} else {
quid = uid;
}
metaReaderClear(&mr);
pSW = metaGetTableSchema(pMeta, quid, sver, 0);
tdInitTSchemaBuilder(&sb, 0);
for (int i = 0; i < pSW->nCols; i++) {
pSchema = pSW->pSchema + i;
tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes);
}
pTSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
taosMemoryFree(pSW->pSchema);
taosMemoryFree(pSW);
return pTSchema;
}
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
#if 0
#ifdef META_TDB_SMA_TEST
STSmaWrapper *pSW = NULL;
pSW = taosMemoryCalloc(1, sizeof(*pSW));
if (pSW == NULL) {
return NULL;
}
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
if (pCur == NULL) {
taosMemoryFree(pSW);
return NULL;
}
void *pBuf = NULL;
SSmaIdxKey *pSmaIdxKey = NULL;
while (true) {
// TODO: lock during iterate?
if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
pSmaIdxKey = pCur->pKey;
ASSERT(pSmaIdxKey != NULL);
void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false);
if (pSmaVal == NULL) {
tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid);
continue;
}
++pSW->number;
STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma));
if (tptr == NULL) {
TDB_FREE(pSmaVal);
metaCloseSmaCursor(pCur);
tdDestroyTSmaWrapper(pSW);
taosMemoryFreeClear(pSW);
return NULL;
}
pSW->tSma = tptr;
pBuf = pSmaVal;
if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
TDB_FREE(pSmaVal);
metaCloseSmaCursor(pCur);
tdDestroyTSmaWrapper(pSW);
taosMemoryFreeClear(pSW);
return NULL;
}
TDB_FREE(pSmaVal);
continue;
}
break;
}
metaCloseSmaCursor(pCur);
return pSW;
#endif
#endif
return NULL;
}
int metaGetTbNum(SMeta *pMeta) {
// TODO
// ASSERT(0);
return 0;
}
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
#if 0
// TODO
// ASSERT(0); // comment this line to pass CI
// return NULL:
#ifdef META_TDB_SMA_TEST
SArray *pUids = NULL;
SMetaDB *pDB = pMeta->pDB;
void *pKey;
// TODO: lock?
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0);
if (pCur == NULL) {
return NULL;
}
// TODO: lock?
SSmaIdxKey *pSmaIdxKey = NULL;
tb_uid_t uid = 0;
while (true) {
// TODO: lock during iterate?
if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
ASSERT(pSmaIdxKey != NULL);
pSmaIdxKey = pCur->pKey;
if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) {
continue;
}
uid = pSmaIdxKey->uid;
if (!pUids) {
pUids = taosArrayInit(16, sizeof(tb_uid_t));
if (!pUids) {
metaCloseSmaCursor(pCur);
return NULL;
}
}
taosArrayPush(pUids, &uid);
continue;
}
break;
}
metaCloseSmaCursor(pCur);
return pUids;
#endif
#endif
return NULL;
}
void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
#if 0
// TODO
// ASSERT(0);
// return NULL;
#ifdef META_TDB_SMA_TEST
SMetaDB *pDB = pMeta->pDB;
void *pKey = NULL;
void *pVal = NULL;
int kLen = 0;
int vLen = 0;
int ret = -1;
// Set key
pKey = (void *)&indexUid;
kLen = sizeof(indexUid);
// Query
ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen);
if (ret != 0 || !pVal) {
return NULL;
}
if (!isDecode) {
// return raw value
return pVal;
}
// Decode
STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma));
if (pCfg == NULL) {
taosMemoryFree(pVal);
return NULL;
}
void *pBuf = pVal;
if (tDecodeTSma(pBuf, pCfg) == NULL) {
tdDestroyTSma(pCfg);
taosMemoryFree(pCfg);
TDB_FREE(pVal);
return NULL;
}
TDB_FREE(pVal);
return pCfg;
#endif
#endif
return NULL;
}
#endif

311
source/dnode/vnode/src/meta/metaTDBImpl.c
View File

@ -15,6 +15,8 @@
#include "vnodeInt.h"
#ifndef META_REFACT
typedef struct SPoolMem {
int64_t size;
struct SPoolMem *prev;
@ -45,7 +47,7 @@ struct SMetaDB {
#endif
};
#pragma pack(push,1)
#pragma pack(push, 1)
typedef struct {
tb_uid_t uid;
int32_t sver;
@ -379,64 +381,6 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
return 0;
}
STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
int ret;
SMetaDB *pMetaDb = pMeta->pDB;
void *pKey;
void *pVal;
int kLen;
int vLen;
STbCfg *pTbCfg;
// Fetch
pKey = &uid;
kLen = sizeof(uid);
pVal = NULL;
ret = tdbDbGet(pMetaDb->pTbDB, pKey, kLen, &pVal, &vLen);
if (ret < 0) {
return NULL;
}
// Decode
pTbCfg = taosMemoryMalloc(sizeof(*pTbCfg));
metaDecodeTbInfo(pVal, pTbCfg);
TDB_FREE(pVal);
return pTbCfg;
}
STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
void *pKey;
void *pVal;
void *ppKey;
int pkLen;
int kLen;
int vLen;
int ret;
pKey = tbname;
kLen = strlen(tbname) + 1;
pVal = NULL;
ppKey = NULL;
ret = tdbDbPGet(pMeta->pDB->pNameIdx, pKey, kLen, &ppKey, &pkLen, &pVal, &vLen);
if (ret < 0) {
return NULL;
}
ASSERT(pkLen == kLen + sizeof(uid));
*uid = *(tb_uid_t *)POINTER_SHIFT(ppKey, kLen);
TDB_FREE(ppKey);
TDB_FREE(pVal);
return metaGetTbInfoByUid(pMeta, *uid);
}
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
}
static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {
void *pKey;
void *pVal;
@ -468,156 +412,6 @@ static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_
return pSchemaWrapper;
}
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
tb_uid_t quid;
SSchemaWrapper *pSW;
STSchemaBuilder sb;
SSchema *pSchema;
STSchema *pTSchema;
STbCfg *pTbCfg;
pTbCfg = metaGetTbInfoByUid(pMeta, uid);
if (pTbCfg->type == META_CHILD_TABLE) {
quid = pTbCfg->ctbCfg.suid;
} else {
quid = uid;
}
pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true);
if (pSW == NULL) {
return NULL;
}
tdInitTSchemaBuilder(&sb, 0);
for (int i = 0; i < pSW->nCols; i++) {
pSchema = pSW->pSchema + i;
tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes);
}
pTSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
return pTSchema;
}
struct SMTbCursor {
TDBC *pDbc;
};
SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
SMTbCursor *pTbCur = NULL;
SMetaDB *pDB = pMeta->pDB;
pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur));
if (pTbCur == NULL) {
return NULL;
}
tdbDbcOpen(pDB->pTbDB, &pTbCur->pDbc);
return pTbCur;
}
void metaCloseTbCursor(SMTbCursor *pTbCur) {
if (pTbCur) {
if (pTbCur->pDbc) {
tdbDbcClose(pTbCur->pDbc);
}
taosMemoryFree(pTbCur);
}
}
char *metaTbCursorNext(SMTbCursor *pTbCur) {
void *pKey = NULL;
void *pVal = NULL;
int kLen;
int vLen;
int ret;
void *pBuf;
STbCfg tbCfg;
for (;;) {
ret = tdbDbNext(pTbCur->pDbc, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break;
pBuf = pVal;
metaDecodeTbInfo(pBuf, &tbCfg);
if (tbCfg.type == META_SUPER_TABLE) {
taosMemoryFree(tbCfg.name);
taosMemoryFree(tbCfg.stbCfg.pTagSchema);
continue;
} else if (tbCfg.type == META_CHILD_TABLE) {
kvRowFree(tbCfg.ctbCfg.pTag);
}
return tbCfg.name;
}
return NULL;
}
struct SMCtbCursor {
TDBC *pCur;
tb_uid_t suid;
void *pKey;
void *pVal;
int kLen;
int vLen;
};
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
SMCtbCursor *pCtbCur = NULL;
SMetaDB *pDB = pMeta->pDB;
int ret;
pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur));
if (pCtbCur == NULL) {
return NULL;
}
pCtbCur->suid = uid;
ret = tdbDbcOpen(pDB->pCtbIdx, &pCtbCur->pCur);
if (ret < 0) {
taosMemoryFree(pCtbCur);
return NULL;
}
// TODO: move the cursor to the suid there
return pCtbCur;
}
void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
if (pCtbCur) {
if (pCtbCur->pCur) {
tdbDbcClose(pCtbCur->pCur);
TDB_FREE(pCtbCur->pKey);
TDB_FREE(pCtbCur->pVal);
}
taosMemoryFree(pCtbCur);
}
}
tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
int ret;
SCtbIdxKey *pCtbIdxKey;
ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen);
if (ret < 0) {
return 0;
}
pCtbIdxKey = pCtbCur->pKey;
return pCtbIdxKey->uid;
}
int metaGetTbNum(SMeta *pMeta) {
// TODO
// ASSERT(0);
return 0;
}
struct SMSmaCursor {
TDBC *pCur;
tb_uid_t uid;
@ -768,53 +562,6 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
return 0;
}
void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
// TODO
// ASSERT(0);
// return NULL;
#ifdef META_TDB_SMA_TEST
SMetaDB *pDB = pMeta->pDB;
void *pKey = NULL;
void *pVal = NULL;
int kLen = 0;
int vLen = 0;
int ret = -1;
// Set key
pKey = (void *)&indexUid;
kLen = sizeof(indexUid);
// Query
ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen);
if (ret != 0 || !pVal) {
return NULL;
}
if (!isDecode) {
// return raw value
return pVal;
}
// Decode
STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma));
if (pCfg == NULL) {
taosMemoryFree(pVal);
return NULL;
}
void *pBuf = pVal;
if (tDecodeTSma(pBuf, pCfg) == NULL) {
tdDestroyTSma(pCfg);
taosMemoryFree(pCfg);
TDB_FREE(pVal);
return NULL;
}
TDB_FREE(pVal);
return pCfg;
#endif
}
/**
* @brief
*
@ -889,56 +636,6 @@ void metaCloseSmaCursor(SMSmaCursor *pCur) {
#endif
}
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
// TODO
// ASSERT(0); // comment this line to pass CI
// return NULL:
#ifdef META_TDB_SMA_TEST
SArray *pUids = NULL;
SMetaDB *pDB = pMeta->pDB;
void *pKey;
// TODO: lock?
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0);
if (pCur == NULL) {
return NULL;
}
// TODO: lock?
SSmaIdxKey *pSmaIdxKey = NULL;
tb_uid_t uid = 0;
while (true) {
// TODO: lock during iterate?
if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
ASSERT(pSmaIdxKey != NULL);
pSmaIdxKey = pCur->pKey;
if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) {
continue;
}
uid = pSmaIdxKey->uid;
if (!pUids) {
pUids = taosArrayInit(16, sizeof(tb_uid_t));
if (!pUids) {
metaCloseSmaCursor(pCur);
return NULL;
}
}
taosArrayPush(pUids, &uid);
continue;
}
break;
}
metaCloseSmaCursor(pCur);
return pUids;
#endif
}
static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) {
int tlen = 0;
SSchema *pSchema;
@ -1142,3 +839,5 @@ static void poolFree(void *arg, void *ptr) {
tdbOsFree(pMem);
}
#endif

272
source/dnode/vnode/src/meta/metaTable.c
View File

@ -15,29 +15,120 @@
#include "vnodeInt.h"
int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
// Validate the tbOptions
// if (metaValidateTbCfg(pMeta, pTbCfg) < 0) {
// // TODO: handle error
// return -1;
// }
static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME);
// TODO: add atomicity
int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SMetaEntry me = {0};
int kLen = 0;
int vLen = 0;
const void *pKey = NULL;
const void *pVal = NULL;
void *pBuf = NULL;
int32_t szBuf = 0;
void *p = NULL;
SCoder coder = {0};
SMetaReader mr = {0};
if (metaSaveTableToDB(pMeta, pTbCfg, pHandle) < 0) {
// TODO: handle error
// validate req
metaReaderInit(&mr, pMeta->pVnode, 0);
if (metaGetTableEntryByName(&mr, pReq->name) == 0) {
// TODO: just for pass case
#if 0
terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
metaReaderClear(&mr);
return -1;
#else
metaReaderClear(&mr);
return 0;
#endif
}
metaReaderClear(&mr);
if (metaSaveTableToIdx(pMeta, pTbCfg) < 0) {
// TODO: handle error
// set structs
me.version = version;
me.type = TSDB_SUPER_TABLE;
me.uid = pReq->suid;
me.name = pReq->name;
me.stbEntry.schema = pReq->schema;
me.stbEntry.schemaTag = pReq->schemaTag;
if (metaHandleEntry(pMeta, &me) < 0) goto _err;
metaDebug("vgId: %d super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
return 0;
_err:
metaError("vgId: %d failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name,
pReq->suid, tstrerror(terrno));
return -1;
}
}
int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
// TODO
return 0;
}
int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
SMetaEntry me = {0};
SMetaReader mr = {0};
// validate message
if (pReq->type != TSDB_CHILD_TABLE && pReq->type != TSDB_NORMAL_TABLE) {
terrno = TSDB_CODE_INVALID_MSG;
goto _err;
}
// preprocess req
pReq->uid = tGenIdPI64();
pReq->ctime = taosGetTimestampSec();
// validate req
metaReaderInit(&mr, pMeta->pVnode, 0);
if (metaGetTableEntryByName(&mr, pReq->name) == 0) {
terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
metaReaderClear(&mr);
return -1;
}
metaReaderClear(&mr);
// build SMetaEntry
me.version = version;
me.type = pReq->type;
me.uid = pReq->uid;
me.name = pReq->name;
if (me.type == TSDB_CHILD_TABLE) {
me.ctbEntry.ctime = pReq->ctime;
me.ctbEntry.ttlDays = pReq->ttl;
me.ctbEntry.suid = pReq->ctb.suid;
me.ctbEntry.pTags = pReq->ctb.pTag;
} else {
me.ntbEntry.ctime = pReq->ctime;
me.ntbEntry.ttlDays = pReq->ttl;
me.ntbEntry.schema = pReq->ntb.schema;
}
if (metaHandleEntry(pMeta, &me) < 0) goto _err;
metaDebug("vgId:%d table %s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid,
pReq->type);
return 0;
_err:
metaError("vgId:%d failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name,
pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno));
return -1;
}
int metaDropTable(SMeta *pMeta, tb_uid_t uid) {
#if 0
if (metaRemoveTableFromIdx(pMeta, uid) < 0) {
// TODO: handle error
return -1;
@ -47,7 +138,164 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid) {
// TODO
return -1;
}
#endif
return 0;
}
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
STbDbKey tbDbKey;
void *pKey = NULL;
void *pVal = NULL;
int kLen = 0;
int vLen = 0;
SCoder coder = {0};
// set key and value
tbDbKey.version = pME->version;
tbDbKey.uid = pME->uid;
pKey = &tbDbKey;
kLen = sizeof(tbDbKey);
if (tEncodeSize(metaEncodeEntry, pME, vLen) < 0) {
goto _err;
}
pVal = taosMemoryMalloc(vLen);
if (pVal == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER);
if (metaEncodeEntry(&coder, pME) < 0) {
goto _err;
}
tCoderClear(&coder);
// write to table.db
if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
goto _err;
}
taosMemoryFree(pVal);
return 0;
_err:
taosMemoryFree(pVal);
return -1;
}
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
}
static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
}
static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
int32_t ttlDays;
int64_t ctime;
STtlIdxKey ttlKey;
if (pME->type == TSDB_CHILD_TABLE) {
ctime = pME->ctbEntry.ctime;
ttlDays = pME->ctbEntry.ttlDays;
} else if (pME->type == TSDB_NORMAL_TABLE) {
ctime = pME->ntbEntry.ctime;
ttlDays = pME->ntbEntry.ttlDays;
} else {
ASSERT(0);
}
if (ttlDays <= 0) return 0;
ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60;
ttlKey.uid = pME->uid;
return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
}
static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
}
static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) {
// TODO
return 0;
}
static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
SCoder coder = {0};
void *pVal = NULL;
int vLen = 0;
int rcode = 0;
SSkmDbKey skmDbKey = {0};
const SSchemaWrapper *pSW;
if (pME->type == TSDB_SUPER_TABLE) {
pSW = &pME->stbEntry.schema;
} else if (pME->type == TSDB_NORMAL_TABLE) {
pSW = &pME->ntbEntry.schema;
} else {
ASSERT(0);
}
skmDbKey.uid = pME->uid;
skmDbKey.sver = pSW->sver;
// encode schema
if (tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen) < 0) return -1;
pVal = taosMemoryMalloc(vLen);
if (pVal == NULL) {
rcode = -1;
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER);
tEncodeSSchemaWrapper(&coder, pSW);
if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) {
rcode = -1;
goto _exit;
}
_exit:
taosMemoryFree(pVal);
tCoderClear(&coder);
return rcode;
}
static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
// save to table.db
if (metaSaveToTbDb(pMeta, pME) < 0) return -1;
// update uid.idx
if (metaUpdateUidIdx(pMeta, pME) < 0) return -1;
// update name.idx
if (metaUpdateNameIdx(pMeta, pME) < 0) return -1;
if (pME->type == TSDB_CHILD_TABLE) {
// update ctb.idx
if (metaUpdateCtbIdx(pMeta, pME) < 0) return -1;
// update tag.idx
if (metaUpdateTagIdx(pMeta, pME) < 0) return -1;
} else {
// update schema.db
if (metaSaveToSkmDb(pMeta, pME) < 0) return -1;
}
if (pME->type != TSDB_SUPER_TABLE) {
if (metaUpdateTtlIdx(pMeta, pME) < 0) return -1;
}
return 0;
}

7
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -142,12 +142,13 @@ int tsdbPrepareCommit(STsdb *pTsdb) {
}
int tsdbCommit(STsdb *pRepo) {
STsdbMemTable *pMem = pRepo->imem;
SCommitH commith = {0};
SDFileSet *pSet = NULL;
int fid;
if (pRepo->imem == NULL) return 0;
// if (pRepo->imem == NULL) return 0;
pRepo->imem = pRepo->mem;
pRepo->mem = NULL;
tsdbStartCommit(pRepo);
// Resource initialization
@ -239,7 +240,7 @@ static void tsdbStartCommit(STsdb *pRepo) {
static void tsdbEndCommit(STsdb *pTsdb, int eno) {
tsdbEndFSTxn(pTsdb);
tsdbFreeMemTable(pTsdb, pTsdb->imem);
tsdbMemTableDestroy(pTsdb, pTsdb->imem);
pTsdb->imem = NULL;
tsdbInfo("vgId:%d commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
}

26
source/dnode/vnode/src/tsdb/tsdbCommit2.c Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
int tsdbBegin(STsdb *pTsdb) {
STsdbMemTable *pMem;
if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) {
return -1;
}
return 0;
}

108
source/dnode/vnode/src/tsdb/tsdbMain.c
View File

@ -15,94 +15,60 @@
#include "vnodeInt.h"
static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF);
static void tsdbFree(STsdb *pTsdb);
static int tsdbOpenImpl(STsdb *pTsdb);
static void tsdbCloseImpl(STsdb *pTsdb);
STsdb *tsdbOpen(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) {
int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) {
STsdb *pTsdb = NULL;
int slen = 0;
// Set default TSDB Options
// if (pTsdbCfg == NULL) {
pTsdbCfg = &defautlTsdbOptions;
// }
*ppTsdb = NULL;
slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_TSDB_DIR) + 3;
// Validate the options
if (tsdbValidateOptions(pTsdbCfg) < 0) {
// TODO: handle error
return NULL;
}
// Create the handle
pTsdb = tsdbNew(path, pVnode, pTsdbCfg, pMAF);
// create handle
pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
if (pTsdb == NULL) {
// TODO: handle error
return NULL;
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
taosMkDir(path);
// Open the TSDB
if (tsdbOpenImpl(pTsdb) < 0) {
// TODO: handle error
return NULL;
}
return pTsdb;
}
void tsdbClose(STsdb *pTsdb) {
if (pTsdb) {
tsdbCloseImpl(pTsdb);
tsdbFree(pTsdb);
}
}
/* ------------------------ STATIC METHODS ------------------------ */
static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) {
STsdb *pTsdb = NULL;
pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb));
if (pTsdb == NULL) {
// TODO: handle error
return NULL;
}
pTsdb->path = strdup(path);
pTsdb->vgId = TD_VID(pVnode);
pTsdb->path = (char *)&pTsdb[1];
sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP,
VNODE_TSDB_DIR);
pTsdb->pVnode = pVnode;
tsdbOptionsCopy(&(pTsdb->config), pTsdbCfg);
pTsdb->pmaf = pMAF;
pTsdb->fs = tsdbNewFS(pTsdbCfg);
pTsdb->vgId = TD_VID(pVnode);
pTsdb->repoLocked = false;
tdbMutexInit(&pTsdb->mutex, NULL);
pTsdb->config = pVnode->config.tsdbCfg;
pTsdb->fs = tsdbNewFS(&pTsdb->config);
return pTsdb;
// create dir (TODO: use tfsMkdir)
taosMkDir(pTsdb->path);
// open tsdb
if (tsdbOpenFS(pTsdb) < 0) {
goto _err;
}
tsdbDebug("vgId: %d tsdb is opened", TD_VID(pVnode));
*ppTsdb = pTsdb;
return 0;
_err:
taosMemoryFree(pTsdb);
return -1;
}
static void tsdbFree(STsdb *pTsdb) {
int tsdbClose(STsdb *pTsdb) {
if (pTsdb) {
tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb));
tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb));
tsdbCloseFS(pTsdb);
// tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb));
// tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb));
tsdbFreeFS(pTsdb->fs);
taosMemoryFreeClear(pTsdb->path);
// taosMemoryFreeClear(pTsdb->path);
taosMemoryFree(pTsdb);
}
}
static int tsdbOpenImpl(STsdb *pTsdb) {
tsdbOpenFS(pTsdb);
// tsdbInitSma(pTsdb);
// TODO
return 0;
}
static void tsdbCloseImpl(STsdb *pTsdb) {
tsdbCloseFS(pTsdb);
// TODO
}
int tsdbLockRepo(STsdb *pTsdb) {
int code = taosThreadMutexLock(&pTsdb->mutex);
if (code != 0) {

188
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -15,8 +15,6 @@
#include "vnodeInt.h"
static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);
static int tsdbMemTableInsertTbData(STsdb *pRepo, SSubmitBlk *pBlock, int32_t *pAffectedRows);
static STbData *tsdbNewTbData(tb_uid_t uid);
static void tsdbFreeTbData(STbData *pTbData);
static char *tsdbGetTsTupleKey(const void *data);
@ -24,86 +22,51 @@ static int tsdbTbDataComp(const void *arg1, const void *arg2);
static char *tsdbTbDataGetUid(const void *arg);
static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row);
STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) {
STsdbMemTable *pMemTable = (STsdbMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable));
int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) {
STsdbMemTable *pMemTable;
SVnode *pVnode;
*ppMemTable = NULL;
pVnode = pTsdb->pVnode;
// alloc handle
pMemTable = (STsdbMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable));
if (pMemTable == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
return -1;
}
pMemTable->pPool = pTsdb->pVnode->inUse;
T_REF_INIT_VAL(pMemTable, 1);
taosInitRWLatch(&(pMemTable->latch));
pMemTable->keyMax = TSKEY_MIN;
taosInitRWLatch(&pMemTable->latch);
pMemTable->keyMin = TSKEY_MAX;
pMemTable->keyMax = TSKEY_MIN;
pMemTable->nRow = 0;
pMemTable->pMA = pTsdb->pmaf->create(pTsdb->pmaf);
if (pMemTable->pMA == NULL) {
taosMemoryFree(pMemTable);
return NULL;
}
// Initialize the container
pMemTable->pSlIdx =
tSkipListCreate(5, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t), tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid);
pMemTable->pSlIdx = tSkipListCreate(pVnode->config.tsdbCfg.slLevel, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t),
tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid);
if (pMemTable->pSlIdx == NULL) {
pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA);
taosMemoryFree(pMemTable);
return NULL;
return -1;
}
pMemTable->pHashIdx = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (pMemTable->pHashIdx == NULL) {
pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA);
tSkipListDestroy(pMemTable->pSlIdx);
taosMemoryFree(pMemTable);
return NULL;
return -1;
}
return pMemTable;
*ppMemTable = pMemTable;
return 0;
}
void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) {
void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable) {
if (pMemTable) {
taosHashCleanup(pMemTable->pHashIdx);
tSkipListDestroy(pMemTable->pSlIdx);
if (pMemTable->pMA) {
pTsdb->pmaf->destroy(pTsdb->pmaf, pMemTable->pMA);
}
taosMemoryFree(pMemTable);
}
}
int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
SSubmitBlk *pBlock = NULL;
SSubmitMsgIter msgIter = {0};
int32_t affectedrows = 0, numOfRows = 0;
if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) {
if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
return -1;
}
tInitSubmitMsgIter(pMsg, &msgIter);
while (true) {
tGetSubmitMsgNext(&msgIter, &pBlock);
if (pBlock == NULL) break;
if (tsdbMemTableInsertTbData(pTsdb, pBlock, &affectedrows) < 0) {
return -1;
}
numOfRows += pBlock->numOfRows;
}
if (pRsp != NULL) {
pRsp->affectedRows = affectedrows;
pRsp->numOfRows = numOfRows;
}
return 0;
}
/**
* This is an important function to load data or try to load data from memory skiplist iterator.
*
@ -255,78 +218,7 @@ int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) {
return 0;
}
static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
ASSERT(pMsg != NULL);
// STsdbMeta * pMeta = pTsdb->tsdbMeta;
SSubmitMsgIter msgIter = {0};
SSubmitBlk *pBlock = NULL;
SSubmitBlkIter blkIter = {0};
STSRow *row = NULL;
TSKEY now = taosGetTimestamp(pTsdb->config.precision);
TSKEY minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep2;
TSKEY maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.days;
terrno = TSDB_CODE_SUCCESS;
pMsg->length = htonl(pMsg->length);
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
while (true) {
if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
if (pBlock == NULL) break;
pBlock->uid = htobe64(pBlock->uid);
pBlock->suid = htobe64(pBlock->suid);
pBlock->sversion = htonl(pBlock->sversion);
pBlock->dataLen = htonl(pBlock->dataLen);
pBlock->schemaLen = htonl(pBlock->schemaLen);
pBlock->numOfRows = htons(pBlock->numOfRows);
#if 0
if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
pBlock->tid);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
return -1;
}
STable *pTable = pMeta->tables[pBlock->tid];
if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
pBlock->tid);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
return -1;
}
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable));
terrno = TSDB_CODE_TDB_INVALID_ACTION;
return -1;
}
// Check schema version and update schema if needed
if (tsdbCheckTableSchema(pTsdb, pBlock, pTable) < 0) {
if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
continue;
} else {
return -1;
}
}
tsdbInitSubmitBlkIter(pBlock, &blkIter);
while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) {
return -1;
}
}
#endif
}
if (terrno != TSDB_CODE_SUCCESS) return -1;
return 0;
}
static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) {
int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) {
// STsdbMeta *pMeta = pRepo->tsdbMeta;
// int32_t points = 0;
// STable *pTable = NULL;
@ -337,11 +229,9 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
STSRow *row;
TSKEY keyMin;
TSKEY keyMax;
SSubmitBlk *pBlkCopy;
// SMemTable *pMemTable = NULL;
// STableData *pTableData = NULL;
// STsdbCfg *pCfg = &(pRepo->config);
// create container is nedd
tptr = taosHashGet(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid));
if (tptr == NULL) {
pTbData = tsdbNewTbData(pBlock->uid);
@ -358,7 +248,11 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
pTbData = *(STbData **)tptr;
}
tInitSubmitBlkIter(pBlock, &blkIter);
// copy data to buffer pool
pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pBlock->dataLen + sizeof(*pBlock));
memcpy(pBlkCopy, pBlock, pBlock->dataLen + sizeof(*pBlock));
tInitSubmitBlkIter(pBlkCopy, &blkIter);
if (blkIter.row == NULL) return 0;
keyMin = TD_ROW_KEY(blkIter.row);
@ -377,31 +271,6 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
(*pAffectedRows) += pBlock->numOfRows;
// STSRow* lastRow = NULL;
// int64_t osize = SL_SIZE(pTableData->pData);
// tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow);
// tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext);
// int64_t dsize = SL_SIZE(pTableData->pData) - osize;
// (*pAffectedRows) += points;
// if(lastRow != NULL) {
// TSKEY lastRowKey = TD_ROW_KEY(lastRow);
// if (pMemTable->keyFirst > firstRowKey) pMemTable->keyFirst = firstRowKey;
// pMemTable->numOfRows += dsize;
// if (pTableData->keyFirst > firstRowKey) pTableData->keyFirst = firstRowKey;
// pTableData->numOfRows += dsize;
// if (pMemTable->keyLast < lastRowKey) pMemTable->keyLast = lastRowKey;
// if (pTableData->keyLast < lastRowKey) pTableData->keyLast = lastRowKey;
// if (tsdbUpdateTableLatestInfo(pRepo, pTable, lastRow) < 0) {
// return -1;
// }
// }
// STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion, -1);
// pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
// pRepo->stat.totalStorage += points * schemaVLen(pSchema);
return 0;
}
@ -526,7 +395,6 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow* row);
static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
static STSRow* tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitReq *pMsg);
static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows);
static int tsdbInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter);
static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);

39
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -618,8 +618,8 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr
return pNew;
}
tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
uint64_t taskId) {
tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList,
uint64_t qId, uint64_t taskId) {
STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList);
if (pNew->numOfTables == 0) {
@ -1187,9 +1187,10 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
if ((ascScan && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || (!ascScan && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
if ((ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || (!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
if ((ascScan && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
(!ascScan && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
if ((ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
(!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
// do not load file block into buffer
int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
@ -1226,7 +1227,8 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
assert(pTsdbReadHandle->outputCapacity >= binfo.rows);
int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &binfo);
if ((cur->pos == 0 && endPos == binfo.rows - 1 && ascScan) || (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ascScan))) {
if ((cur->pos == 0 && endPos == binfo.rows - 1 && ascScan) ||
(cur->pos == (binfo.rows - 1) && endPos == 0 && (!ascScan))) {
pTsdbReadHandle->realNumOfRows = binfo.rows;
cur->rows = binfo.rows;
@ -1773,7 +1775,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
STable* pTable = NULL;
int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo);
tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64 " rows:%d, start:%d, end:%d, %s",
tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64
" rows:%d, start:%d, end:%d, %s",
pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows,
cur->pos, endPos, pTsdbReadHandle->idStr);
@ -3613,8 +3616,11 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd
int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) {
STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid);
if (pTbCfg == NULL) {
SMetaReader mr = {0};
metaReaderInit(&mr, ((SMeta*)pMeta)->pVnode, 0);
if (metaGetTableEntryByUid(&mr, uid) < 0) {
tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
goto _error;
@ -3622,13 +3628,15 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
}
if (pTbCfg->type != META_SUPER_TABLE) {
if (mr.me.type != META_SUPER_TABLE) {
tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId,
reqId);
terrno = TSDB_CODE_OPS_NOT_SUPPORT; // basically, this error is caused by invalid sql issued by client
goto _error;
}
metaReaderClear(&mr);
// NOTE: not add ref count for super table
SArray* res = taosArrayInit(8, sizeof(STableKeyInfo));
SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true);
@ -3679,12 +3687,18 @@ int32_t tsdbQueryTableList(void* pMeta, SArray* pRes, void* filterInfo) {
return TSDB_CODE_SUCCESS;
}
int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) {
STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid);
if (pTbCfg == NULL) {
SMeta* metaP = (SMeta*)pMeta;
SMetaReader mr = {0};
metaReaderInit(&mr, metaP->pVnode, 0);
if (metaGetTableEntryByUid(&mr, uid) < 0) {
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
goto _error;
}
metaReaderClear(&mr);
pGroupInfo->numOfTables = 1;
pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
@ -3697,6 +3711,7 @@ int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGr
return TSDB_CODE_SUCCESS;
_error:
metaReaderClear(&mr);
return terrno;
}

2
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -1693,6 +1693,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) {
* @return int32_t
*/
int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) {
#if 0
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
if (!param) {
@ -1764,6 +1765,7 @@ int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) {
tsdbDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, REPO_ID(pTsdb), pReq->stbCfg.suid);
}
#endif
return TSDB_CODE_SUCCESS;
}

118
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -15,21 +15,111 @@
#include "vnodeInt.h"
/**
* @brief insert TS data
*
* @param pTsdb
* @param pMsg
* @param pRsp
* @return int
*/
int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
// Check if mem is there. If not, create one.
if (pTsdb->mem == NULL) {
pTsdb->mem = tsdbNewMemTable(pTsdb);
if (pTsdb->mem == NULL) {
static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);
int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
SSubmitMsgIter msgIter = {0};
SSubmitBlk *pBlock = NULL;
int32_t affectedrows = 0;
int32_t numOfRows = 0;
ASSERT(pTsdb->mem != NULL);
// scan and convert
if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) {
if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
return -1;
}
// loop to insert
tInitSubmitMsgIter(pMsg, &msgIter);
while (true) {
tGetSubmitMsgNext(&msgIter, &pBlock);
if (pBlock == NULL) break;
if (tsdbInsertTableData(pTsdb, pBlock, &affectedrows) < 0) {
return -1;
}
numOfRows += pBlock->numOfRows;
}
if (pRsp != NULL) {
pRsp->affectedRows = affectedrows;
pRsp->numOfRows = numOfRows;
}
return 0;
}
static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
ASSERT(pMsg != NULL);
// STsdbMeta * pMeta = pTsdb->tsdbMeta;
SSubmitMsgIter msgIter = {0};
SSubmitBlk *pBlock = NULL;
SSubmitBlkIter blkIter = {0};
STSRow *row = NULL;
TSKEY now = taosGetTimestamp(pTsdb->config.precision);
TSKEY minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep2;
TSKEY maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.days;
terrno = TSDB_CODE_SUCCESS;
pMsg->length = htonl(pMsg->length);
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
while (true) {
if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
if (pBlock == NULL) break;
pBlock->uid = htobe64(pBlock->uid);
pBlock->suid = htobe64(pBlock->suid);
pBlock->sversion = htonl(pBlock->sversion);
pBlock->dataLen = htonl(pBlock->dataLen);
pBlock->schemaLen = htonl(pBlock->schemaLen);
pBlock->numOfRows = htons(pBlock->numOfRows);
#if 0
if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
pBlock->tid);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
return -1;
}
STable *pTable = pMeta->tables[pBlock->tid];
if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid,
pBlock->tid);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
return -1;
}
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable));
terrno = TSDB_CODE_TDB_INVALID_ACTION;
return -1;
}
// Check schema version and update schema if needed
if (tsdbCheckTableSchema(pTsdb, pBlock, pTable) < 0) {
if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
continue;
} else {
return -1;
}
}
return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, pRsp);
tsdbInitSubmitBlkIter(pBlock, &blkIter);
while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
if (tsdbCheckRowRange(pTsdb, pTable, row, minKey, maxKey, now) < 0) {
return -1;
}
}
#endif
}
if (terrno != TSDB_CODE_SUCCESS) return -1;
return 0;
}

117
source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c
View File

@ -1,117 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
static SVArenaNode *vArenaNodeNew(uint64_t capacity);
static void vArenaNodeFree(SVArenaNode *pNode);
SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) {
SVMemAllocator *pVMA = (SVMemAllocator *)taosMemoryMalloc(sizeof(*pVMA));
if (pVMA == NULL) {
return NULL;
}
pVMA->capacity = capacity;
pVMA->ssize = ssize;
pVMA->lsize = lsize;
TD_SLIST_INIT(&(pVMA->nlist));
pVMA->pNode = vArenaNodeNew(capacity);
if (pVMA->pNode == NULL) {
taosMemoryFree(pVMA);
return NULL;
}
TD_SLIST_PUSH(&(pVMA->nlist), pVMA->pNode);
return pVMA;
}
void vmaDestroy(SVMemAllocator *pVMA) {
if (pVMA) {
while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) {
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
TD_SLIST_POP(&(pVMA->nlist));
vArenaNodeFree(pNode);
}
taosMemoryFree(pVMA);
}
}
void vmaReset(SVMemAllocator *pVMA) {
while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) {
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
TD_SLIST_POP(&(pVMA->nlist));
vArenaNodeFree(pNode);
}
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
pNode->ptr = pNode->data;
}
void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) {
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
void * ptr;
if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) {
uint64_t capacity = TMAX(pVMA->ssize, size);
pNode = vArenaNodeNew(capacity);
if (pNode == NULL) {
// TODO: handle error
return NULL;
}
TD_SLIST_PUSH(&(pVMA->nlist), pNode);
}
ptr = pNode->ptr;
pNode->ptr = POINTER_SHIFT(ptr, size);
return ptr;
}
void vmaFree(SVMemAllocator *pVMA, void *ptr) {
// TODO
}
bool vmaIsFull(SVMemAllocator *pVMA) {
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) ||
(pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize);
}
/* ------------------------ STATIC METHODS ------------------------ */
static SVArenaNode *vArenaNodeNew(uint64_t capacity) {
SVArenaNode *pNode = NULL;
pNode = (SVArenaNode *)taosMemoryMalloc(sizeof(*pNode) + capacity);
if (pNode == NULL) {
return NULL;
}
pNode->size = capacity;
pNode->ptr = pNode->data;
return pNode;
}
static void vArenaNodeFree(SVArenaNode *pNode) {
if (pNode) {
taosMemoryFree(pNode);
}
}

14
source/dnode/vnode/src/vnd/vnodeBufferPool2.c → source/dnode/vnode/src/vnd/vnodeBufPool.c
View File

@ -17,7 +17,7 @@
/* ------------------------ STRUCTURES ------------------------ */
static int vnodeBufPoolCreate(int size, SVBufPool **ppPool);
static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool);
static int vnodeBufPoolDestroy(SVBufPool *pPool);
int vnodeOpenBufPool(SVnode *pVnode, int64_t size) {
@ -30,17 +30,17 @@ int vnodeOpenBufPool(SVnode *pVnode, int64_t size) {
// create pool
ret = vnodeBufPoolCreate(size, &pPool);
if (ret < 0) {
vError("vgId:%d failed to open vnode buffer pool since %s", TD_VNODE_ID(pVnode), tstrerror(terrno));
vError("vgId:%d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno));
vnodeCloseBufPool(pVnode);
return -1;
}
// add pool to queue
// add pool to vnode
pPool->next = pVnode->pPool;
pVnode->pPool = pPool;
}
vDebug("vgId:%d vnode buffer pool is opened, pool size: %" PRId64, TD_VNODE_ID(pVnode), size);
vDebug("vgId:%d vnode buffer pool is opened, pool size: %" PRId64, TD_VID(pVnode), size);
return 0;
}
@ -53,7 +53,7 @@ int vnodeCloseBufPool(SVnode *pVnode) {
vnodeBufPoolDestroy(pPool);
}
vDebug("vgId:%d vnode buffer pool is closed", TD_VNODE_ID(pVnode));
vDebug("vgId:%d vnode buffer pool is closed", TD_VID(pVnode));
return 0;
}
@ -75,7 +75,7 @@ void vnodeBufPoolReset(SVBufPool *pPool) {
pPool->ptr = pPool->node.data;
}
void *vnodeBufPoolMalloc(SVBufPool *pPool, size_t size) {
void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
SVBufPoolNode *pNode;
void *p;
@ -120,7 +120,7 @@ void vnodeBufPoolFree(SVBufPool *pPool, void *p) {
}
// STATIC METHODS -------------------
static int vnodeBufPoolCreate(int size, SVBufPool **ppPool) {
static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool) {
SVBufPool *pPool;
pPool = taosMemoryMalloc(sizeof(SVBufPool) + size);

191
source/dnode/vnode/src/vnd/vnodeBufferPool.c
View File

@ -1,191 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
/* ------------------------ STRUCTURES ------------------------ */
#define VNODE_BUF_POOL_SHARDS 3
struct SVBufPool {
TdThreadMutex mutex;
TdThreadCond hasFree;
TD_DLIST(SVMemAllocator) free;
TD_DLIST(SVMemAllocator) incycle;
SVMemAllocator *inuse;
// MAF for submodules to use
SMemAllocatorFactory *pMAF;
};
static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF);
static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA);
int vnodeOpenBufPool(SVnode *pVnode) {
uint64_t capacity;
if ((pVnode->pBufPool = (SVBufPool *)taosMemoryCalloc(1, sizeof(SVBufPool))) == NULL) {
/* TODO */
return -1;
}
TD_DLIST_INIT(&(pVnode->pBufPool->free));
TD_DLIST_INIT(&(pVnode->pBufPool->incycle));
pVnode->pBufPool->inuse = NULL;
// TODO
capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS;
for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) {
SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize);
if (pVMA == NULL) {
// TODO: handle error
return -1;
}
TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA);
}
pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)taosMemoryMalloc(sizeof(SMemAllocatorFactory));
if (pVnode->pBufPool->pMAF == NULL) {
// TODO: handle error
return -1;
}
pVnode->pBufPool->pMAF->impl = pVnode;
pVnode->pBufPool->pMAF->create = vBufPoolCreateMA;
pVnode->pBufPool->pMAF->destroy = vBufPoolDestroyMA;
return 0;
}
void vnodeCloseBufPool(SVnode *pVnode) {
if (pVnode->pBufPool) {
taosMemoryFreeClear(pVnode->pBufPool->pMAF);
vmaDestroy(pVnode->pBufPool->inuse);
while (true) {
SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle));
if (pVMA == NULL) break;
TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA);
vmaDestroy(pVMA);
}
while (true) {
SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free));
if (pVMA == NULL) break;
TD_DLIST_POP(&(pVnode->pBufPool->free), pVMA);
vmaDestroy(pVMA);
}
taosMemoryFree(pVnode->pBufPool);
pVnode->pBufPool = NULL;
}
}
int vnodeBufPoolSwitch(SVnode *pVnode) {
SVMemAllocator *pvma = pVnode->pBufPool->inuse;
pVnode->pBufPool->inuse = NULL;
if (pvma) {
TD_DLIST_APPEND(&(pVnode->pBufPool->incycle), pvma);
}
return 0;
}
int vnodeBufPoolRecycle(SVnode *pVnode) {
SVBufPool * pBufPool = pVnode->pBufPool;
SVMemAllocator *pvma = TD_DLIST_HEAD(&(pBufPool->incycle));
if (pvma == NULL) return 0;
// ASSERT(pvma != NULL);
TD_DLIST_POP(&(pBufPool->incycle), pvma);
vmaReset(pvma);
TD_DLIST_APPEND(&(pBufPool->free), pvma);
return 0;
}
void *vnodeMalloc(SVnode *pVnode, uint64_t size) {
SVBufPool *pBufPool = pVnode->pBufPool;
if (pBufPool->inuse == NULL) {
while (true) {
// TODO: add sem_wait and sem_post
pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free));
if (pBufPool->inuse) {
TD_DLIST_POP(&(pBufPool->free), pBufPool->inuse);
break;
} else {
// tsem_wait(&(pBufPool->hasFree));
}
}
}
return vmaMalloc(pBufPool->inuse, size);
}
bool vnodeBufPoolIsFull(SVnode *pVnode) {
if (pVnode->pBufPool->inuse == NULL) return false;
return vmaIsFull(pVnode->pBufPool->inuse);
}
SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode) { return pVnode->pBufPool->pMAF; }
/* ------------------------ STATIC METHODS ------------------------ */
typedef struct {
SVnode * pVnode;
SVMemAllocator *pVMA;
} SVMAWrapper;
static FORCE_INLINE void *vmaMaloocCb(SMemAllocator *pMA, uint64_t size) {
SVMAWrapper *pWrapper = (SVMAWrapper *)(pMA->impl);
return vmaMalloc(pWrapper->pVMA, size);
}
// TODO: Add atomic operations here
static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) {
SMemAllocator *pMA;
SVnode * pVnode = (SVnode *)(pMAF->impl);
SVMAWrapper * pWrapper;
pMA = (SMemAllocator *)taosMemoryCalloc(1, sizeof(*pMA) + sizeof(SVMAWrapper));
if (pMA == NULL) {
return NULL;
}
pVnode->pBufPool->inuse->_ref.val++;
pWrapper = POINTER_SHIFT(pMA, sizeof(*pMA));
pWrapper->pVnode = pVnode;
pWrapper->pVMA = pVnode->pBufPool->inuse;
pMA->impl = pWrapper;
TD_MA_MALLOC_FUNC(pMA) = vmaMaloocCb;
return pMA;
}
static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) {
SVMAWrapper * pWrapper = (SVMAWrapper *)(pMA->impl);
SVnode * pVnode = pWrapper->pVnode;
SVMemAllocator *pVMA = pWrapper->pVMA;
taosMemoryFree(pMA);
if (--pVMA->_ref.val == 0) {
TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA);
vmaReset(pVMA);
TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA);
}
}

35
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -19,17 +19,18 @@ const SVnodeCfg vnodeCfgDefault = {
.vgId = -1,
.dbname = "",
.dbId = 0,
.wsize = 96 * 1024 * 1024,
.ssize = 1 * 1024 * 1024,
.lsize = 1024,
.isHeapAllocator = false,
.szPage = 4096,
.szCache = 256,
.szBuf = 96 * 1024 * 1024,
.isHeap = false,
.ttl = 0,
.keep = 0,
.streamMode = 0,
.isWeak = 0,
.tsdbCfg = {.precision = TWO_STAGE_COMP,
.tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI,
.update = 0,
.compression = 2,
.slLevel = 5,
.days = 10,
.minRows = 100,
.maxRows = 4096,
@ -53,10 +54,10 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1;
if (tjsonAddStringToObject(pJson, "dbname", pCfg->dbname) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wsize", pCfg->wsize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "ssize", pCfg->ssize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "lsize", pCfg->lsize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "ttl", pCfg->ttl) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep", pCfg->keep) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
@ -64,6 +65,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1;
@ -103,10 +105,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (tjsonGetNumberValue(pJson, "vgId", pCfg->vgId) < 0) return -1;
if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1;
if (tjsonGetNumberValue(pJson, "dbId", pCfg->dbId) < 0) return -1;
if (tjsonGetNumberValue(pJson, "wsize", pCfg->wsize) < 0) return -1;
if (tjsonGetNumberValue(pJson, "ssize", pCfg->ssize) < 0) return -1;
if (tjsonGetNumberValue(pJson, "lsize", pCfg->lsize) < 0) return -1;
if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1;
if (tjsonGetNumberValue(pJson, "szPage", pCfg->szPage) < 0) return -1;
if (tjsonGetNumberValue(pJson, "szCache", pCfg->szCache) < 0) return -1;
if (tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
if (tjsonGetNumberValue(pJson, "ttl", pCfg->ttl) < 0) return -1;
if (tjsonGetNumberValue(pJson, "keep", pCfg->keep) < 0) return -1;
if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
@ -114,6 +116,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
if (tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
if (tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1;
if (tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1;
if (tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1;
if (tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1;
if (tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1;
@ -150,10 +153,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
return 0;
}
int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) {
int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) {
uint32_t hashValue = 0;
switch (pVnodeOptions->hashMethod) {
switch (pVnode->config.hashMethod) {
default:
hashValue = MurmurHash3_32(tableFName, strlen(tableFName));
break;
@ -167,5 +170,5 @@ int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) {
}
#endif
return TSDB_CODE_SUCCESS;
return 0;
}

95
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -22,9 +22,41 @@ static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData);
static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo);
static int vnodeStartCommit(SVnode *pVnode);
static int vnodeEndCommit(SVnode *pVnode);
static int vnodeCommit(void *arg);
static int vnodeCommitImpl(void *arg);
static void vnodeWaitCommit(SVnode *pVnode);
int vnodeBegin(SVnode *pVnode) {
// alloc buffer pool
/* pthread_mutex_lock(); */
while (pVnode->pPool == NULL) {
/* pthread_cond_wait(); */
}
pVnode->inUse = pVnode->pPool;
pVnode->pPool = pVnode->inUse->next;
pVnode->inUse->next = NULL;
/* ref pVnode->inUse buffer pool */
/* pthread_mutex_unlock(); */
// begin meta
if (metaBegin(pVnode->pMeta) < 0) {
vError("vgId: %d failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
// begin tsdb
if (tsdbBegin(pVnode->pTsdb) < 0) {
vError("vgId: %d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
return 0;
}
int vnodeShouldCommit(SVnode *pVnode) { return pVnode->inUse->size > pVnode->config.szBuf / 3; }
int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile;
@ -142,10 +174,10 @@ _err:
int vnodeAsyncCommit(SVnode *pVnode) {
vnodeWaitCommit(pVnode);
vnodeBufPoolSwitch(pVnode);
// vnodeBufPoolSwitch(pVnode);
tsdbPrepareCommit(pVnode->pTsdb);
vnodeScheduleTask(vnodeCommit, pVnode);
vnodeScheduleTask(vnodeCommitImpl, pVnode);
return 0;
}
@ -157,25 +189,64 @@ int vnodeSyncCommit(SVnode *pVnode) {
return 0;
}
static int vnodeCommit(void *arg) {
SVnode *pVnode = (SVnode *)arg;
int vnodeCommit(SVnode *pVnode) {
SVnodeInfo info;
char dir[TSDB_FILENAME_LEN];
SVnodeInfo info = {0};
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
vInfo("vgId:%d start to commit, version: %" PRId64, TD_VID(pVnode), pVnode->state.applied);
pVnode->onCommit = pVnode->inUse;
pVnode->inUse = NULL;
// save info
info.config = pVnode->config;
info.state.committed = pVnode->state.applied;
info.state.applied = pVnode->state.applied;
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
if (vnodeSaveInfo(dir, &info) < 0) {
ASSERT(0);
return -1;
}
vnodeSaveInfo(dir, &info);
// commit each sub-system
if (metaCommit(pVnode->pMeta) < 0) {
ASSERT(0);
return -1;
}
if (tsdbCommit(pVnode->pTsdb) < 0) {
ASSERT(0);
return -1;
}
if (tqCommit(pVnode->pTq) < 0) {
ASSERT(0);
return -1;
}
// walCommit (TODO)
// commit info
if (vnodeCommitInfo(dir, &info) < 0) {
ASSERT(0);
return -1;
}
// apply the commit (TODO)
vnodeBufPoolReset(pVnode->onCommit);
pVnode->onCommit->next = pVnode->pPool;
pVnode->pPool = pVnode->onCommit;
pVnode->onCommit = NULL;
vInfo("vgId:%d commit over", TD_VID(pVnode));
return 0;
}
static int vnodeCommitImpl(void *arg) {
SVnode *pVnode = (SVnode *)arg;
// metaCommit(pVnode->pMeta);
tqCommit(pVnode->pTq);
tsdbCommit(pVnode->pTsdb);
vnodeCommitInfo(dir, &info);
vnodeBufPoolRecycle(pVnode);
// vnodeBufPoolRecycle(pVnode);
tsem_post(&(pVnode->canCommit));
return 0;
}

27
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -85,7 +85,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
tsem_init(&(pVnode->canCommit), 0, 1);
// open buffer pool
if (vnodeOpenBufPool(pVnode) < 0) {
if (vnodeOpenBufPool(pVnode, pVnode->config.isHeap ? 0 : pVnode->config.szBuf / 3) < 0) {
vError("vgId: %d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
@ -97,9 +97,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
}
// open tsdb
sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TSDB_DIR);
pVnode->pTsdb = tsdbOpen(tdir, pVnode, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode));
if (pVnode->pTsdb == NULL) {
if (tsdbOpen(pVnode, &pVnode->pTsdb) < 0) {
vError("vgId: %d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
@ -126,17 +124,17 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
goto _err;
}
// sync integration
// open sync
if (vnodeSyncOpen(pVnode, dir)) {
// vnode begin
if (vnodeBegin(pVnode) < 0) {
vError("vgId: %d failed to begin since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
#if 0
if (vnodeBegin() < 0) {
// open sync
if (vnodeSyncOpen(pVnode, dir)) {
vError("vgId: %d failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
#endif
return pVnode;
@ -153,14 +151,9 @@ _err:
void vnodeClose(SVnode *pVnode) {
if (pVnode) {
// commit (TODO: use option to control)
vnodeSyncCommit(pVnode);
// close vnode
vnodeQueryClose(pVnode);
// sync integration
vnodeCommit(pVnode);
vnodeSyncClose(pVnode);
vnodeQueryClose(pVnode);
walClose(pVnode->pWal);
tqClose(pVnode->pTq);
tsdbClose(pVnode->pTsdb);

138
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -22,94 +22,76 @@ int vnodeQueryOpen(SVnode *pVnode) {
void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); }
int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
STbCfg * pTbCfg = NULL;
STbCfg * pStbCfg = NULL;
tb_uid_t uid;
int32_t nCols;
int32_t nTagCols;
SSchemaWrapper *pSW = NULL;
STableMetaRsp * pTbMetaMsg = NULL;
STableMetaRsp metaRsp = {0};
SSchema * pTagSchema;
SRpcMsg rpcMsg;
int msgLen = 0;
int32_t code = 0;
char tableFName[TSDB_TABLE_FNAME_LEN];
int32_t rspLen = 0;
void * pRsp = NULL;
STableInfoReq infoReq = {0};
STableMetaRsp metaRsp = {0};
SMetaReader mer1 = {0};
SMetaReader mer2 = {0};
char tableFName[TSDB_TABLE_FNAME_LEN];
SRpcMsg rpcMsg;
int32_t code = 0;
int32_t rspLen = 0;
void *pRsp = NULL;
SSchemaWrapper schema = {0};
SSchemaWrapper schemaTag = {0};
// decode req
if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
metaRsp.dbId = pVnode->config.dbId;
memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
strcpy(metaRsp.tbName, infoReq.tbName);
memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName);
code = vnodeValidateTableHash(&pVnode->config, tableFName);
code = vnodeValidateTableHash(pVnode, tableFName);
if (code) {
goto _exit;
}
pTbCfg = metaGetTbInfoByName(pVnode->pMeta, infoReq.tbName, &uid);
if (pTbCfg == NULL) {
code = TSDB_CODE_VND_TB_NOT_EXIST;
// query meta
metaReaderInit(&mer1, pVnode, 0);
if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) {
goto _exit;
}
if (pTbCfg->type == META_CHILD_TABLE) {
pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid);
if (pStbCfg == NULL) {
code = TSDB_CODE_VND_TB_NOT_EXIST;
goto _exit;
}
pSW = metaGetTableSchema(pVnode->pMeta, pTbCfg->ctbCfg.suid, 0, true);
} else {
pSW = metaGetTableSchema(pVnode->pMeta, uid, 0, true);
}
nCols = pSW->nCols;
if (pTbCfg->type == META_SUPER_TABLE) {
nTagCols = pTbCfg->stbCfg.nTagCols;
pTagSchema = pTbCfg->stbCfg.pTagSchema;
} else if (pTbCfg->type == META_CHILD_TABLE) {
nTagCols = pStbCfg->stbCfg.nTagCols;
pTagSchema = pStbCfg->stbCfg.pTagSchema;
} else {
nTagCols = 0;
pTagSchema = NULL;
}
metaRsp.pSchemas = taosMemoryCalloc(nCols + nTagCols, sizeof(SSchema));
if (metaRsp.pSchemas == NULL) {
code = TSDB_CODE_VND_OUT_OF_MEMORY;
goto _exit;
}
if (pTbCfg->type == META_CHILD_TABLE) {
strcpy(metaRsp.stbName, pStbCfg->name);
metaRsp.suid = pTbCfg->ctbCfg.suid;
} else if (pTbCfg->type == META_SUPER_TABLE) {
strcpy(metaRsp.stbName, pTbCfg->name);
metaRsp.suid = uid;
}
metaRsp.numOfTags = nTagCols;
metaRsp.numOfColumns = nCols;
metaRsp.tableType = pTbCfg->type;
metaRsp.tuid = uid;
metaRsp.tableType = mer1.me.type;
metaRsp.vgId = TD_VID(pVnode);
metaRsp.tuid = mer1.me.uid;
memcpy(metaRsp.pSchemas, pSW->pSchema, sizeof(SSchema) * pSW->nCols);
if (nTagCols) {
memcpy(POINTER_SHIFT(metaRsp.pSchemas, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols);
if (mer1.me.type == TSDB_SUPER_TABLE) {
strcpy(metaRsp.stbName, mer1.me.name);
schema = mer1.me.stbEntry.schema;
schemaTag = mer1.me.stbEntry.schemaTag;
metaRsp.suid = mer1.me.uid;
} else if (mer1.me.type == TSDB_CHILD_TABLE) {
metaReaderInit(&mer2, pVnode, 0);
if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
strcpy(metaRsp.stbName, mer2.me.name);
metaRsp.suid = mer2.me.uid;
schema = mer2.me.stbEntry.schema;
schemaTag = mer2.me.stbEntry.schemaTag;
} else if (mer1.me.type == TSDB_NORMAL_TABLE) {
schema = mer1.me.ntbEntry.schema;
} else {
ASSERT(0);
}
_exit:
metaRsp.numOfTags = schemaTag.nCols;
metaRsp.numOfColumns = schema.nCols;
metaRsp.precision = pVnode->config.tsdbCfg.precision;
metaRsp.sversion = schema.sver;
metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags));
memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
if (schemaTag.nCols) {
memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
}
// encode and send response
rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
if (rspLen < 0) {
code = TSDB_CODE_INVALID_MSG;
@ -123,23 +105,6 @@ _exit:
}
tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
tFreeSTableMetaRsp(&metaRsp);
if (pSW != NULL) {
taosMemoryFreeClear(pSW->pSchema);
taosMemoryFreeClear(pSW);
}
if (pTbCfg) {
taosMemoryFreeClear(pTbCfg->name);
if (pTbCfg->type == META_SUPER_TABLE) {
taosMemoryFree(pTbCfg->stbCfg.pTagSchema);
} else if (pTbCfg->type == META_SUPER_TABLE) {
kvRowFree(pTbCfg->ctbCfg.pTag);
}
taosMemoryFreeClear(pTbCfg);
}
rpcMsg.handle = pMsg->handle;
rpcMsg.ahandle = pMsg->ahandle;
rpcMsg.refId = pMsg->refId;
@ -148,7 +113,12 @@ _exit:
rpcMsg.code = code;
tmsgSendRsp(&rpcMsg);
return TSDB_CODE_SUCCESS;
_exit:
taosMemoryFree(metaRsp.pSchemas);
metaReaderClear(&mer2);
metaReaderClear(&mer1);
return code;
}
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {

282
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -17,10 +17,13 @@
#include "syncTools.h"
#include "vnodeInt.h"
static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq);
static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg *pRsp);
static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq);
static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp);
static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp);
static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
#if 0
@ -47,44 +50,55 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) {
void *ptr = NULL;
void *pReq;
int len;
int ret;
if (pVnode->config.streamMode == 0) {
ptr = vnodeMalloc(pVnode, pMsg->contLen);
if (ptr == NULL) {
// TODO: handle error
}
vTrace("vgId: %d start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
version);
// TODO: copy here need to be extended
memcpy(ptr, pMsg->pCont, pMsg->contLen);
}
pVnode->state.applied = version;
// skip header
pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
len = pMsg->contLen - sizeof(SMsgHead);
// todo: change the interface here
if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
// TODO: handle error
vError("vgId: %d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
switch (pMsg->msgType) {
/* META */
case TDMT_VND_CREATE_STB:
ret = vnodeProcessCreateStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)));
break;
case TDMT_VND_CREATE_TABLE:
pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
vnodeProcessCreateTbReq(pVnode, pMsg, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pRsp);
if (vnodeProcessCreateStbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_ALTER_STB:
vnodeProcessAlterStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)));
if (vnodeProcessAlterStbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_DROP_STB:
vTrace("vgId:%d, process drop stb req", TD_VID(pVnode));
if (vnodeProcessDropStbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_CREATE_TABLE:
if (vnodeProcessCreateTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_ALTER_TABLE:
if (vnodeProcessAlterTbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_DROP_TABLE:
if (vnodeProcessDropTbReq(pVnode, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_CREATE_SMA: { // timeRangeSMA
if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
// TODO
}
} break;
/* TSDB */
case TDMT_VND_SUBMIT:
tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, ptr, STREAM_DATA_TYPE_SUBMIT_BLOCK);
pRsp->msgType = TDMT_VND_SUBMIT_RSP;
vnodeProcessSubmitReq(pVnode, ptr, pRsp);
if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err;
break;
/* TQ */
case TDMT_VND_MQ_VG_CHANGE:
if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
pMsg->contLen - sizeof(SMsgHead)) < 0) {
@ -101,19 +115,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
0) < 0) {
}
} break;
case TDMT_VND_CREATE_SMA: { // timeRangeSMA
if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
// TODO
}
// } break;
// case TDMT_VND_CANCEL_SMA: { // timeRangeSMA
// } break;
// case TDMT_VND_DROP_SMA: { // timeRangeSMA
// if (tsdbDropTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
// // TODO
// }
} break;
case TDMT_VND_ALTER_VNODE:
break;
default:
@ -121,17 +122,24 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
break;
}
pVnode->state.applied = version;
vDebug("vgId: %d process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version);
// Check if it needs to commit
// commit if need
if (vnodeShouldCommit(pVnode)) {
// tsem_wait(&(pVnode->canCommit));
if (vnodeAsyncCommit(pVnode) < 0) {
// TODO: handle error
}
vInfo("vgId:%d commit at version %" PRId64, TD_VID(pVnode), version);
// commit current change
vnodeCommit(pVnode);
// start a new one
vnodeBegin(pVnode);
}
return 0;
_err:
vDebug("vgId: %d process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
tstrerror(terrno), version);
return -1;
}
int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
@ -279,95 +287,111 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
return 0;
}
static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) {
SVCreateTbReq vCreateTbReq = {0};
tDeserializeSVCreateTbReq(pReq, &vCreateTbReq);
if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq), NULL) < 0) {
// TODO
static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
SVCreateStbReq req = {0};
SCoder coder;
pRsp->msgType = TDMT_VND_CREATE_STB_RSP;
pRsp->code = TSDB_CODE_SUCCESS;
pRsp->pCont = NULL;
pRsp->contLen = 0;
// decode and process req
tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER);
if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
pRsp->code = terrno;
goto _err;
}
// tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &vCreateTbReq);
if (metaCreateSTable(pVnode->pMeta, version, &req) < 0) {
pRsp->code = terrno;
goto _err;
}
tCoderClear(&coder);
return 0;
_err:
tCoderClear(&coder);
return -1;
}
tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &vCreateTbReq);
taosMemoryFree(vCreateTbReq.stbCfg.pSchema);
taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema);
if (vCreateTbReq.stbCfg.pRSmaParam) {
taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg1);
taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg2);
taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam);
}
taosMemoryFree(vCreateTbReq.name);
return 0;
}
static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg *pRsp) {
SVCreateTbBatchReq vCreateTbBatchReq = {0};
SVCreateTbBatchRsp vCreateTbBatchRsp = {0};
tDeserializeSVCreateTbBatchReq(pReq, &vCreateTbBatchReq);
int reqNum = taosArrayGetSize(vCreateTbBatchReq.pArray);
static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) {
SCoder coder = {0};
int rcode = 0;
SVCreateTbBatchReq req = {0};
SVCreateTbReq *pCreateReq;
SVCreateTbBatchRsp rsp = {0};
SVCreateTbRsp cRsp = {0};
char tbName[TSDB_TABLE_FNAME_LEN];
STbDdlH ddlHandle = {
.ahandle = pVnode->pTsdb,
.result = NULL,
.fp = tsdbFetchTbUidList,
};
pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
pRsp->code = TSDB_CODE_SUCCESS;
pRsp->pCont = NULL;
pRsp->contLen = 0;
for (int i = 0; i < reqNum; i++) {
SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i);
char tableFName[TSDB_TABLE_FNAME_LEN];
SMsgHead *pHead = (SMsgHead *)pMsg->pCont;
sprintf(tableFName, "%s.%s", pVnode->config.dbname, pCreateTbReq->name);
int32_t code = vnodeValidateTableHash(&pVnode->config, tableFName);
if (code) {
SVCreateTbRsp rsp;
rsp.code = code;
taosArrayPush(vCreateTbBatchRsp.rspList, &rsp);
// decode
tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER);
if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) {
rcode = -1;
terrno = TSDB_CODE_INVALID_MSG;
goto _exit;
}
if (metaCreateTable(pVnode->pMeta, pCreateTbReq, &ddlHandle) < 0) {
// TODO: handle error
vError("vgId:%d, failed to create table: %s", TD_VID(pVnode), pCreateTbReq->name);
rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs);
if (rsp.pArray == NULL) {
rcode = -1;
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
// TODO: to encapsule a free API
taosMemoryFree(pCreateTbReq->name);
if (pCreateTbReq->type == TD_SUPER_TABLE) {
taosMemoryFree(pCreateTbReq->stbCfg.pSchema);
taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema);
if (pCreateTbReq->stbCfg.pRSmaParam) {
taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam);
// loop to create table
for (int iReq = 0; iReq < req.nReqs; iReq++) {
pCreateReq = req.pReqs + iReq;
// validate hash
sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name);
if (vnodeValidateTableHash(pVnode, tbName) < 0) {
cRsp.code = TSDB_CODE_VND_HASH_MISMATCH;
taosArrayPush(rsp.pArray, &cRsp);
continue;
}
} else if (pCreateTbReq->type == TD_CHILD_TABLE) {
taosMemoryFree(pCreateTbReq->ctbCfg.pTag);
// do create table
if (metaCreateTable(pVnode->pMeta, version, pCreateReq) < 0) {
cRsp.code = terrno;
} else {
taosMemoryFree(pCreateTbReq->ntbCfg.pSchema);
if (pCreateTbReq->ntbCfg.pRSmaParam) {
taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam);
}
}
cRsp.code = TSDB_CODE_SUCCESS;
}
tsdbUpdateTbUidList(pVnode->pTsdb, ddlHandle.result);
vTrace("vgId:%d process create %" PRIzu " tables", TD_VID(pVnode), taosArrayGetSize(vCreateTbBatchReq.pArray));
taosArrayDestroy(vCreateTbBatchReq.pArray);
if (vCreateTbBatchRsp.rspList) {
int32_t contLen = tSerializeSVCreateTbBatchRsp(NULL, 0, &vCreateTbBatchRsp);
void *msg = rpcMallocCont(contLen);
tSerializeSVCreateTbBatchRsp(msg, contLen, &vCreateTbBatchRsp);
taosArrayDestroy(vCreateTbBatchRsp.rspList);
pRsp->pCont = msg;
pRsp->contLen = contLen;
taosArrayPush(rsp.pArray, &cRsp);
}
return 0;
tCoderClear(&coder);
// prepare rsp
tEncodeSize(tEncodeSVCreateTbBatchRsp, &rsp, pRsp->contLen);
pRsp->pCont = rpcMallocCont(pRsp->contLen);
if (pRsp->pCont == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
rcode = -1;
goto _exit;
}
tCoderInit(&coder, TD_LITTLE_ENDIAN, pRsp->pCont, pRsp->contLen, TD_ENCODER);
tEncodeSVCreateTbBatchRsp(&coder, &rsp);
_exit:
taosArrayClear(rsp.pArray);
tCoderClear(&coder);
return rcode;
}
static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) {
static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
// ASSERT(0);
#if 0
SVCreateTbReq vAlterTbReq = {0};
vTrace("vgId:%d, process alter stb req", TD_VID(pVnode));
tDeserializeSVCreateTbReq(pReq, &vAlterTbReq);
@ -378,20 +402,44 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) {
taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam);
}
taosMemoryFree(vAlterTbReq.name);
#endif
return 0;
}
static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp) {
static int vnodeProcessDropStbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
// TODO
// ASSERT(0);
return 0;
}
static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
// TODO
ASSERT(0);
return 0;
}
static int vnodeProcessDropTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcMsg *pRsp) {
// TODO
ASSERT(0);
return 0;
}
static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
SSubmitReq *pSubmitReq = (SSubmitReq *)pReq;
SSubmitRsp rsp = {0};
pRsp->code = 0;
// handle the request
if (tsdbInsertData(pVnode->pTsdb, pSubmitReq, &rsp) < 0) {
if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) {
pRsp->code = terrno;
return -1;
}
// pRsp->msgType = TDMT_VND_SUBMIT_RSP;
// vnodeProcessSubmitReq(pVnode, ptr, pRsp);
// tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, ptr, STREAM_DATA_TYPE_SUBMIT_BLOCK);
// encode the response (TODO)
pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp));
memcpy(pRsp->pCont, &rsp, sizeof(rsp));

5
source/libs/executor/inc/executorimpl.h
View File

@ -12,6 +12,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// clang-format off
#ifndef TDENGINE_EXECUTORIMPL_H
#define TDENGINE_EXECUTORIMPL_H
@ -38,6 +39,8 @@ extern "C" {
#include "tmsg.h"
#include "tpagedbuf.h"
#include "vnode.h"
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED)
@ -385,7 +388,7 @@ typedef struct SSysTableScanInfo {
int32_t accountId;
bool showRewrite;
SNode* pCondition; // db_name filter condition, to discard data that are not in current database
void* pCur; // cursor for iterate the local table meta store.
SMTbCursor* pCur; // cursor for iterate the local table meta store.
SArray* scanCols; // SArray<int16_t> scan column id list
SName name;
SSDataBlock* pRes;

30
source/libs/executor/src/scanoperator.c
View File

@ -245,7 +245,8 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols);
// reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream operator.
// reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream
// operator.
pBlock->info.blockId = 0;
doFilter(pTableScanInfo->pFilterNode, pBlock);
@ -262,7 +263,7 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction
SET_REVERSE_SCAN_FLAG(pTableScanInfo);
switchCtxOrder(pCtx, numOfOutput);
// setupQueryRangeForReverseScan(pTableScanInfo);
// setupQueryRangeForReverseScan(pTableScanInfo);
STimeWindow* pTWindow = &pTableScanInfo->cond.twindow;
TSWAP(pTWindow->skey, pTWindow->ekey, int64_t);
@ -325,7 +326,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
pTableScanInfo->scanFlag = REPEAT_SCAN;
STimeWindow* pWin = &pTableScanInfo->cond.twindow;
qDebug("%s start to repeat ascending order scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
qDebug("%s start to repeat ascending order scan data blocks due to query func required, qrange:%" PRId64
"-%" PRId64,
GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
// do prepare for the next round table scan operation
@ -356,7 +358,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTableScanInfo->scanFlag = REPEAT_SCAN;
qDebug("%s start to repeat descending order scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
qDebug("%s start to repeat descending order scan data blocks due to query func required, qrange:%" PRId64
"-%" PRId64,
GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
// do prepare for the next round table scan operation
@ -369,8 +372,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
return NULL;
}
SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput, int32_t dataLoadFlag, const uint8_t* scanInfo,
SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput,
int32_t dataLoadFlag, const uint8_t* scanInfo, SArray* pColMatchInfo,
SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval,
double sampleRatio, SExecTaskInfo* pTaskInfo) {
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -382,11 +387,11 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCon
}
pInfo->cond = *pCond;
pInfo->scanInfo = (SScanInfo) {.numOfAsc = scanInfo[0], .numOfDesc = scanInfo[1]};
pInfo->scanInfo = (SScanInfo){.numOfAsc = scanInfo[0], .numOfDesc = scanInfo[1]};
pInfo->interval = *pInterval;
pInfo->sampleRatio = sampleRatio;
pInfo->dataBlockLoadFlag= dataLoadFlag;
pInfo->dataBlockLoadFlag = dataLoadFlag;
pInfo->pResBlock = pResBlock;
pInfo->pFilterNode = pCondition;
pInfo->dataReader = pDataReader;
@ -818,8 +823,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
int32_t numOfRows = 0;
char n[TSDB_TABLE_NAME_LEN] = {0};
while ((tb = metaTbCursorNext(pInfo->pCur)) != NULL) {
STR_TO_VARSTR(n, tb);
while (metaTbCursorNext(pInfo->pCur) == 0) {
STR_TO_VARSTR(n, pInfo->pCur->mr.me.name);
colDataAppend(pTableNameCol, numOfRows, n, false);
numOfRows += 1;
if (numOfRows >= pInfo->capacity) {
@ -1119,7 +1124,8 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
pInfo->pRes = blockDataDestroy(pInfo->pRes);
}
SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput,
SExecTaskInfo* pTaskInfo) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -1140,7 +1146,7 @@ SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr,
pOperator->closeFn = destroyTagScanOperatorInfo;
return pOperator;
_error:
_error:
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);
terrno = TSDB_CODE_OUT_OF_MEMORY;

2
source/libs/executor/test/CMakeLists.txt
View File

@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
executorTest
PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes
PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes vnode
)
TARGET_INCLUDE_DIRECTORIES(

7
source/libs/parser/src/parInsert.c
View File

@ -22,6 +22,7 @@
#include "ttime.h"
#include "ttypes.h"
// clang-format off
#define NEXT_TOKEN(pSql, sToken) \
do { \
int32_t index = 0; \
@ -767,8 +768,8 @@ static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const SName* pName, SKVRo
tNameGetFullDbName(pName, dbFName);
pTbReq->type = TD_CHILD_TABLE;
pTbReq->name = strdup(pName->tname);
pTbReq->ctbCfg.suid = suid;
pTbReq->ctbCfg.pTag = row;
pTbReq->ctb.suid = suid;
pTbReq->ctb.pTag = row;
return TSDB_CODE_SUCCESS;
}
@ -1007,7 +1008,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da
void destroyCreateSubTbReq(SVCreateTbReq* pReq) {
taosMemoryFreeClear(pReq->name);
taosMemoryFreeClear(pReq->ctbCfg.pTag);
taosMemoryFreeClear(pReq->ctb.pTag);
}
static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {

19
source/libs/parser/src/parInsertData.c
View File

@ -12,7 +12,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// clang-format off
#include "parInsertData.h"
#include "catalog.h"
@ -157,7 +157,11 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star
}
int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) {
int32_t len = tSerializeSVCreateTbReq(NULL, pCreateTbReq);
SCoder coder = {0};
char* pBuf;
int32_t len;
tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len);
if (pBlocks->nAllocSize - pBlocks->size < len) {
pBlocks->nAllocSize += len + pBlocks->rowSize;
char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize);
@ -169,8 +173,13 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq)
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
}
char* pBuf = pBlocks->pData + pBlocks->size;
tSerializeSVCreateTbReq((void**)&pBuf, pCreateTbReq);
pBuf= pBlocks->pData + pBlocks->size;
tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, len, TD_ENCODER);
tEncodeSVCreateTbReq(&coder, pCreateTbReq);
tCoderClear(&coder);
pBlocks->size += len;
pBlocks->createTbReqLen = len;
return TSDB_CODE_SUCCESS;
@ -191,7 +200,7 @@ int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int3
return ret;
}
if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctbCfg.pTag) {
if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctb.pTag) {
ret = buildCreateTbMsg(*dataBlocks, pCreateTbReq);
if (ret != TSDB_CODE_SUCCESS) {
return ret;

32
source/libs/parser/src/parTranslater.c
View File

@ -3234,7 +3234,7 @@ typedef struct SVgroupTablesBatch {
static void destroyCreateTbReq(SVCreateTbReq* pReq) {
taosMemoryFreeClear(pReq->name);
taosMemoryFreeClear(pReq->ntbCfg.pSchema);
taosMemoryFreeClear(pReq->ntb.schema.pSchema);
}
static int32_t buildSmaParam(STableOptions* pOptions, SVCreateTbReq* pReq) {
@ -3242,12 +3242,14 @@ static int32_t buildSmaParam(STableOptions* pOptions, SVCreateTbReq* pReq) {
return TSDB_CODE_SUCCESS;
}
#if 0
pReq->ntbCfg.pRSmaParam = taosMemoryCalloc(1, sizeof(SRSmaParam));
if (NULL == pReq->ntbCfg.pRSmaParam) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pReq->ntbCfg.pRSmaParam->delay = GET_OPTION_VAL(pOptions->pDelay, TSDB_DEFAULT_DB_DELAY);
pReq->ntbCfg.pRSmaParam->xFilesFactor = GET_OPTION_VAL(pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR);
#endif
return TSDB_CODE_SUCCESS;
}
@ -3262,16 +3264,17 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
SVCreateTbReq req = {0};
req.type = TD_NORMAL_TABLE;
req.name = strdup(pStmt->tableName);
req.ntbCfg.nCols = LIST_LENGTH(pStmt->pCols);
req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema));
if (NULL == req.name || NULL == req.ntbCfg.pSchema) {
req.ntb.schema.nCols = LIST_LENGTH(pStmt->pCols);
req.ntb.schema.sver = 0;
req.ntb.schema.pSchema = taosMemoryCalloc(req.ntb.schema.nCols, sizeof(SSchema));
if (NULL == req.name || NULL == req.ntb.schema.pSchema) {
destroyCreateTbReq(&req);
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pCol;
col_id_t index = 0;
FOREACH(pCol, pStmt->pCols) {
toSchema((SColumnDefNode*)pCol, index + 1, req.ntbCfg.pSchema + index);
toSchema((SColumnDefNode*)pCol, index + 1, req.ntb.schema.pSchema + index);
++index;
}
if (TSDB_CODE_SUCCESS != buildSmaParam(pStmt->pOptions, &req)) {
@ -3292,7 +3295,11 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
}
static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) {
int tlen = sizeof(SMsgHead) + tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req));
int tlen;
SCoder coder = {0};
tEncodeSize(tEncodeSVCreateTbBatchReq, &pTbBatch->req, tlen);
tlen += sizeof(SMsgHead); //+ tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req));
void* buf = taosMemoryMalloc(tlen);
if (NULL == buf) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -3300,7 +3307,10 @@ static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray*
((SMsgHead*)buf)->vgId = htonl(pTbBatch->info.vgId);
((SMsgHead*)buf)->contLen = htonl(tlen);
void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tSerializeSVCreateTbBatchReq(&pBuf, &(pTbBatch->req));
tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, tlen - sizeof(SMsgHead), TD_ENCODER);
tEncodeSVCreateTbBatchReq(&coder, &pTbBatch->req);
tCoderClear(&coder);
SVgDataBlocks* pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
if (NULL == pVgData) {
@ -3322,9 +3332,9 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) {
taosMemoryFreeClear(pTableReq->name);
if (pTableReq->type == TSDB_NORMAL_TABLE) {
taosMemoryFreeClear(pTableReq->ntbCfg.pSchema);
taosMemoryFreeClear(pTableReq->ntb.schema.pSchema);
} else if (pTableReq->type == TSDB_CHILD_TABLE) {
taosMemoryFreeClear(pTableReq->ctbCfg.pTag);
taosMemoryFreeClear(pTableReq->ctb.pTag);
}
}
@ -3405,8 +3415,8 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c
struct SVCreateTbReq req = {0};
req.type = TD_CHILD_TABLE;
req.name = strdup(pTableName);
req.ctbCfg.suid = suid;
req.ctbCfg.pTag = row;
req.ctb.suid = suid;
req.ctb.pTag = row;
SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId));
if (pTableBatch == NULL) {

10
source/libs/scheduler/src/scheduler.c
View File

@ -1076,17 +1076,17 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
SVCreateTbBatchRsp batchRsp = {0};
if (msg) {
SCH_ERR_JRET(tDeserializeSVCreateTbBatchRsp(msg, msgSize, &batchRsp));
if (batchRsp.rspList) {
int32_t num = taosArrayGetSize(batchRsp.rspList);
if (batchRsp.pArray) {
int32_t num = taosArrayGetSize(batchRsp.pArray);
for (int32_t i = 0; i < num; ++i) {
SVCreateTbRsp *rsp = taosArrayGet(batchRsp.rspList, i);
SVCreateTbRsp *rsp = taosArrayGet(batchRsp.pArray, i);
if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) {
taosArrayDestroy(batchRsp.rspList);
taosArrayDestroy(batchRsp.pArray);
SCH_ERR_JRET(rsp->code);
}
}
taosArrayDestroy(batchRsp.rspList);
taosArrayDestroy(batchRsp.pArray);
}
}

115
source/libs/tdb/src/db/tdbBtree.c
View File

@ -42,8 +42,7 @@ struct SBTree {
ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \
TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0))
#pragma pack(push,1)
#pragma pack(push, 1)
typedef struct {
TDB_BTREE_PAGE_COMMON_HDR
} SLeafHdr;
@ -71,8 +70,7 @@ typedef struct {
static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst);
static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2);
static int tdbBtreeOpenImpl(SBTree *pBt);
static int tdbBtreeZeroPage(SPage *pPage, void *arg);
static int tdbBtreeInitPage(SPage *pPage, void *arg);
static int tdbBtreeInitPage(SPage *pPage, void *arg, int init);
static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell,
int *szCell);
static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder);
@ -312,59 +310,37 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
}
// Try to create a new database
SBtreeInitPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt};
ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg, NULL);
ret = tdbPagerAllocPage(pBt->pPager, &pgno);
if (ret < 0) {
ASSERT(0);
return -1;
}
// TODO: here still has problem
tdbPagerReturnPage(pBt->pPager, pPage, NULL);
ASSERT(pgno != 0);
pBt->root = pgno;
return 0;
}
static int tdbBtreeInitPage(SPage *pPage, void *arg) {
static int tdbBtreeInitPage(SPage *pPage, void *arg, int init) {
SBTree *pBt;
u8 flags;
u8 isLeaf;
pBt = (SBTree *)arg;
flags = TDB_BTREE_PAGE_GET_FLAGS(pPage);
isLeaf = TDB_BTREE_PAGE_IS_LEAF(pPage);
ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage));
tdbPageInit(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize);
TDB_BTREE_ASSERT_FLAG(flags);
if (isLeaf) {
pPage->kLen = pBt->keyLen;
pPage->vLen = pBt->valLen;
pPage->maxLocal = pBt->maxLeaf;
pPage->minLocal = pBt->minLeaf;
} else {
pPage->kLen = pBt->keyLen;
pPage->vLen = sizeof(SPgno);
pPage->maxLocal = pBt->maxLocal;
pPage->minLocal = pBt->minLocal;
}
return 0;
}
static int tdbBtreeZeroPage(SPage *pPage, void *arg) {
u8 flags;
SBTree *pBt;
u8 leaf;
flags = ((SBtreeInitPageArg *)arg)->flags;
pBt = ((SBtreeInitPageArg *)arg)->pBt;
if (init) {
// init page
flags = TDB_BTREE_PAGE_GET_FLAGS(pPage);
leaf = TDB_BTREE_PAGE_IS_LEAF(pPage);
TDB_BTREE_ASSERT_FLAG(flags);
tdbPageInit(pPage, leaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize);
} else {
// zero page
flags = ((SBtreeInitPageArg *)arg)->flags;
leaf = flags & TDB_BTREE_LEAF;
TDB_BTREE_ASSERT_FLAG(flags);
tdbPageZero(pPage, leaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize);
@ -372,15 +348,19 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) {
SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pData);
pLeafHdr->flags = flags;
} else {
SIntHdr *pIntHdr = (SIntHdr *)(pPage->pData);
pIntHdr->flags = flags;
pIntHdr->pgno = 0;
}
}
if (leaf) {
pPage->kLen = pBt->keyLen;
pPage->vLen = pBt->valLen;
pPage->maxLocal = pBt->maxLeaf;
pPage->minLocal = pBt->minLeaf;
} else {
SIntHdr *pIntHdr = (SIntHdr *)(pPage->pData);
pIntHdr->flags = flags;
pIntHdr->pgno = 0;
pPage->kLen = pBt->keyLen;
pPage->vLen = sizeof(SPgno);
pPage->maxLocal = pBt->maxLocal;
@ -405,10 +385,11 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN
flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot);
leaf = TDB_BTREE_PAGE_IS_LEAF(pRoot);
// Allocate a new child page
// allocate a new child page
pgnoChild = 0;
zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT);
zArg.pBt = pBt;
ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg, pTxn);
ret = tdbPagerFetchPage(pPager, &pgnoChild, &pChild, tdbBtreeInitPage, &zArg, pTxn);
if (ret < 0) {
return -1;
}
@ -430,7 +411,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN
// Reinitialize the root page
zArg.flags = TDB_BTREE_ROOT;
zArg.pBt = pBt;
ret = tdbBtreeZeroPage(pRoot, &zArg);
ret = tdbBtreeInitPage(pRoot, &zArg, 0);
if (ret < 0) {
return -1;
}
@ -483,7 +464,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
pgno = *(SPgno *)pCell;
}
ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt, pTxn);
ret = tdbPagerFetchPage(pBt->pPager, &pgno, pOlds + i, tdbBtreeInitPage,
&((SBtreeInitPageArg){.pBt = pBt, .flags = 0}), pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
@ -644,9 +626,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
if (iNew < nOlds) {
pNews[iNew] = pOlds[iNew];
} else {
pgno = 0;
iarg.pBt = pBt;
iarg.flags = flags;
ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg, pTxn);
ret = tdbPagerFetchPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeInitPage, &iarg, pTxn);
if (ret < 0) {
ASSERT(0);
}
@ -674,13 +657,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
iarg.pBt = pBt;
iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]);
for (int i = 0; i < nOlds; i++) {
tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], NULL, NULL);
tdbBtreeZeroPage(pOldsCopy[i], &iarg);
tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], tdbDefaultMalloc, NULL);
tdbBtreeInitPage(pOldsCopy[i], &iarg, 0);
tdbPageCopy(pOlds[i], pOldsCopy[i]);
}
iNew = 0;
nNewCells = 0;
tdbBtreeZeroPage(pNews[iNew], &iarg);
tdbBtreeInitPage(pNews[iNew], &iarg, 0);
for (int iOld = 0; iOld < nOlds; iOld++) {
SPage *pPage;
@ -721,7 +704,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
iNew++;
nNewCells = 0;
if (iNew < nNews) {
tdbBtreeZeroPage(pNews[iNew], &iarg);
tdbBtreeInitPage(pNews[iNew], &iarg, 0);
}
}
} else {
@ -740,7 +723,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
iNew++;
nNewCells = 0;
if (iNew < nNews) {
tdbBtreeZeroPage(pNews[iNew], &iarg);
tdbBtreeInitPage(pNews[iNew], &iarg, 0);
}
}
}
@ -760,7 +743,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
}
for (int i = 0; i < nOlds; i++) {
tdbPageDestroy(pOldsCopy[i], NULL, NULL);
tdbPageDestroy(pOldsCopy[i], tdbDefaultFree, NULL);
}
}
@ -1035,7 +1018,13 @@ int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) {
pBtc->iPage = -1;
pBtc->pPage = NULL;
pBtc->idx = -1;
if (pTxn == NULL) {
pBtc->pTxn = &pBtc->txn;
tdbTxnOpen(pBtc->pTxn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
} else {
pBtc->pTxn = pTxn;
}
return 0;
}
@ -1052,7 +1041,8 @@ int tdbBtcMoveToFirst(SBTC *pBtc) {
if (pBtc->iPage < 0) {
// move a clean cursor
ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn);
ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage,
&((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
@ -1117,7 +1107,8 @@ int tdbBtcMoveToLast(SBTC *pBtc) {
if (pBtc->iPage < 0) {
// move a clean cursor
ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn);
ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage,
&((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
@ -1286,13 +1277,16 @@ static int tdbBtcMoveDownward(SBTC *pBtc) {
pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno;
}
ASSERT(pgno);
pBtc->pgStack[pBtc->iPage] = pBtc->pPage;
pBtc->idxStack[pBtc->iPage] = pBtc->idx;
pBtc->iPage++;
pBtc->pPage = NULL;
pBtc->idx = -1;
ret = tdbPagerFetchPage(pBtc->pBt->pPager, pgno, &pBtc->pPage, tdbBtreeInitPage, pBtc->pBt, pBtc->pTxn);
ret = tdbPagerFetchPage(pBtc->pBt->pPager, &pgno, &pBtc->pPage, tdbBtreeInitPage,
&((SBtreeInitPageArg){.pBt = pBtc->pBt, .flags = 0}), pBtc->pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
@ -1327,7 +1321,8 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) {
if (pBtc->iPage < 0) {
// move from a clear cursor
ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn);
ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage,
&((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn);
if (ret < 0) {
// TODO
ASSERT(0);

48
source/libs/tdb/src/db/tdbPCache.c
View File

@ -95,6 +95,8 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
i32 nRef;
ASSERT(pTxn);
nRef = TDB_UNREF_PAGE(pPage);
ASSERT(nRef >= 0);
@ -108,13 +110,12 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
if (pPage->isLocal) {
tdbPCacheUnpinPage(pCache, pPage);
} else {
if (TDB_TXN_IS_WRITE(pTxn)) {
// remove from hash
tdbPCacheRemovePageFromHash(pCache, pPage);
// free the page
if (pTxn && pTxn->xFree) {
tdbPageDestroy(pPage, pTxn->xFree, pTxn->xArg);
}
tdbPageDestroy(pPage, pTxn->xFree, pTxn->xArg);
}
}
@ -125,8 +126,11 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
int ret;
SPage *pPage;
int ret = 0;
SPage *pPage = NULL;
SPage *pPageH = NULL;
ASSERT(pTxn);
// 1. Search the hash table
pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash];
@ -136,11 +140,16 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
}
if (pPage) {
// TODO: the page need to be copied and
// replaced the page in hash table
if (pPage->isLocal || TDB_TXN_IS_WRITE(pTxn)) {
tdbPCachePinPage(pCache, pPage);
return pPage;
}
}
// 1. pPage == NULL
// 2. pPage && pPage->isLocal == 0 && !TDB_TXN_IS_WRITE(pTxn)
pPageH = pPage;
pPage = NULL;
// 2. Try to allocate a new page from the free list
if (pCache->pFree) {
@ -158,7 +167,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
}
// 4. Try a create new page
if (!pPage && pTxn && pTxn->xMalloc) {
if (!pPage) {
ret = tdbPageCreate(pCache->pageSize, &pPage, pTxn->xMalloc, pTxn->xArg);
if (ret < 0) {
// TODO
@ -176,13 +185,28 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
// or by recycling or allocated streesly,
// need to initialize it
if (pPage) {
if (pPageH) {
// copy the page content
memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid));
pPage->pLruNext = NULL;
pPage->pPager = pPageH->pPager;
memcpy(pPage->pData, pPageH->pData, pPage->pageSize);
tdbPageInit(pPage, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize);
pPage->kLen = pPageH->kLen;
pPage->vLen = pPageH->vLen;
pPage->maxLocal = pPageH->maxLocal;
pPage->minLocal = pPageH->minLocal;
} else {
memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid));
pPage->pLruNext = NULL;
pPage->pPager = NULL;
// TODO: allocated page may not add to hash
if (pPage->isLocal || TDB_TXN_IS_WRITE(pTxn)) {
tdbPCacheAddPageToHash(pCache, pPage);
}
}
}
return pPage;
}
@ -249,7 +273,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
pCache->nFree = 0;
pCache->pFree = NULL;
for (int i = 0; i < pCache->cacheSize; i++) {
ret = tdbPageCreate(pCache->pageSize, &pPage, NULL, NULL);
ret = tdbPageCreate(pCache->pageSize, &pPage, tdbDefaultMalloc, NULL);
if (ret < 0) {
// TODO: handle error
return -1;
@ -272,7 +296,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
// Open the hash table
pCache->nPage = 0;
pCache->nHash = pCache->cacheSize;
pCache->nHash = pCache->cacheSize < 8 ? 8 : pCache->cacheSize;
pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *));
if (pCache->pgHash == NULL) {
// TODO

17
source/libs/tdb/src/db/tdbPage.c
View File

@ -43,15 +43,14 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
u8 *ptr;
int size;
ASSERT(xMalloc);
ASSERT(TDB_IS_PGSIZE_VLD(pageSize));
*ppPage = NULL;
size = pageSize + sizeof(*pPage);
if (xMalloc == NULL) {
xMalloc = tdbDefaultMalloc;
}
ptr = (u8 *)((*xMalloc)(arg, size));
ptr = (u8 *)(xMalloc(arg, size));
if (ptr == NULL) {
return -1;
}
@ -75,12 +74,10 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) {
u8 *ptr;
if (!xFree) {
xFree = tdbDefaultFree;
}
ASSERT(xFree);
ptr = pPage->pData;
(*xFree)(arg, ptr);
xFree(arg, ptr);
return 0;
}
@ -436,7 +433,7 @@ static int tdbPageDefragment(SPage *pPage) {
/* ---------------------------------------------------------------------------------------------------------- */
#pragma pack(push,1)
#pragma pack(push, 1)
typedef struct {
u16 cellNum;
u16 cellBody;
@ -520,7 +517,7 @@ SPageMethods pageMethods = {
setPageFreeCellInfo // setFreeCellInfo
};
#pragma pack(push,1)
#pragma pack(push, 1)
typedef struct {
u8 cellNum[3];
u8 cellBody[3];

98
source/libs/tdb/src/db/tdbPager.c
View File

@ -15,7 +15,7 @@
#include "tdbInt.h"
#pragma pack(push,1)
#pragma pack(push, 1)
typedef struct {
u8 hdrString[16];
u16 pageSize;
@ -29,7 +29,8 @@ TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct")
#define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL)
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage);
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *, int), void *arg,
u8 loadPage);
static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
@ -80,6 +81,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
pPager->pageSize = tdbPCacheGetPageSize(pCache);
// pPager->dbOrigSize
ret = tdbGetFileSize(pPager->fd, pPager->pageSize, &(pPager->dbOrigSize));
pPager->dbFileSize = pPager->dbOrigSize;
*ppPager = pPager;
return 0;
@ -228,13 +230,30 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
return 0;
}
int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg,
TXN *pTxn) {
SPage *pPage;
SPgid pgid;
int ret;
SPgno pgno;
u8 loadPage;
// Fetch a page container from the page cache
pgno = *ppgno;
loadPage = 1;
// alloc new page
if (pgno == 0) {
loadPage = 0;
ret = tdbPagerAllocPage(pPager, &pgno);
if (ret < 0) {
ASSERT(0);
return -1;
}
}
ASSERT(pgno > 0);
// fetch a page container
memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
pgid.pgno = pgno;
pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn);
@ -242,9 +261,9 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
return -1;
}
// Initialize the page if need
// init page if need
if (!TDB_PAGE_INITIALIZED(pPage)) {
ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 1);
ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage);
if (ret < 0) {
return -1;
}
@ -253,46 +272,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
ASSERT(TDB_PAGE_INITIALIZED(pPage));
ASSERT(pPage->pPager == pPager);
*ppPage = pPage;
return 0;
}
int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
TXN *pTxn) {
int ret;
SPage *pPage;
SPgid pgid;
// Allocate a page number
ret = tdbPagerAllocPage(pPager, ppgno);
if (ret < 0) {
ASSERT(0);
return -1;
}
ASSERT(*ppgno != 0);
// Fetch a page container from the page cache
memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
pgid.pgno = *ppgno;
pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn);
if (pPage == NULL) {
ASSERT(0);
return -1;
}
ASSERT(!TDB_PAGE_INITIALIZED(pPage));
// Initialize the page if need
ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 0);
if (ret < 0) {
ASSERT(0);
return -1;
}
ASSERT(TDB_PAGE_INITIALIZED(pPage));
ASSERT(pPage->pPager == pPager);
*ppgno = pgno;
*ppPage = pPage;
return 0;
}
@ -333,11 +313,14 @@ int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
return 0;
}
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage) {
static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *, int), void *arg,
u8 loadPage) {
int ret;
int lcode;
int nLoops;
i64 nRead;
SPgno pgno;
int init = 0;
lcode = TDB_TRY_LOCK_PAGE(pPage);
if (lcode == P_LOCK_SUCC) {
@ -346,20 +329,21 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
return 0;
}
if (loadPage) {
nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * TDB_PAGE_PGNO(pPage));
if (nRead < 0) {
// TODO
ASSERT(0);
return -1;
} else if (nRead < pPage->pageSize) {
// TODO
pgno = TDB_PAGE_PGNO(pPage);
if (loadPage && pgno <= pPager->dbOrigSize) {
init = 1;
nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * (pgno - 1));
if (nRead < pPage->pageSize) {
ASSERT(0);
return -1;
}
} else {
init = 0;
}
ret = (*initPage)(pPage, arg);
ret = (*initPage)(pPage, arg, init);
if (ret < 0) {
TDB_UNLOCK_PAGE(pPage);
return -1;
@ -409,7 +393,7 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
i64 offset;
int ret;
offset = pPage->pageSize * TDB_PAGE_PGNO(pPage);
offset = pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
ASSERT(0);
return -1;

1
source/libs/tdb/src/inc/tdbBtree.h
View File

@ -36,6 +36,7 @@ struct SBTC {
int idxStack[BTREE_MAX_DEPTH + 1];
SPage *pgStack[BTREE_MAX_DEPTH + 1];
TXN *pTxn;
TXN txn;
};
// SBTree

4
source/libs/tdb/src/inc/tdbPager.h
View File

@ -42,9 +42,7 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate);
int tdbPagerWrite(SPager *pPager, SPage *pPage);
int tdbPagerBegin(SPager *pPager, TXN *pTxn);
int tdbPagerCommit(SPager *pPager, TXN *pTxn);
int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
TXN *pTxn);
int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg,
TXN *pTxn);
void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);

88
source/libs/tdb/test/tdbTest.cpp
View File

@ -1,5 +1,6 @@
#include <gtest/gtest.h>
#include "os.h"
#include "tdbInt.h"
#include <string>
@ -122,6 +123,8 @@ TEST(tdb_test, simple_test) {
int nData = 10000000;
TXN txn;
taosRemoveDir("tdb");
// Open Env
ret = tdbEnvOpen("tdb", 4096, 64, &pEnv);
GTEST_ASSERT_EQ(ret, 0);
@ -229,3 +232,88 @@ TEST(tdb_test, simple_test) {
ret = tdbEnvClose(pEnv);
GTEST_ASSERT_EQ(ret, 0);
}
TEST(tdb_test, simple_test2) {
int ret;
TENV *pEnv;
TDB *pDb;
FKeyComparator compFunc;
int nData = 1000000;
TXN txn;
taosRemoveDir("tdb");
// Open Env
ret = tdbEnvOpen("tdb", 1024, 10, &pEnv);
GTEST_ASSERT_EQ(ret, 0);
// Create a database
compFunc = tDefaultKeyCmpr;
ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0);
{
char key[64];
char val[64];
int64_t txnid = 0;
SPoolMem *pPool;
// open the pool
pPool = openPool();
// start a transaction
txnid++;
tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
tdbBegin(pEnv, &txn);
for (int iData = 1; iData <= nData; iData++) {
sprintf(key, "key%d", iData);
sprintf(val, "value%d", iData);
ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
GTEST_ASSERT_EQ(ret, 0);
}
{ // Iterate to query the DB data
TDBC *pDBC;
void *pKey = NULL;
void *pVal = NULL;
int vLen, kLen;
int count = 0;
ret = tdbDbcOpen(pDb, &pDBC);
GTEST_ASSERT_EQ(ret, 0);
for (;;) {
ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break;
std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " ";
std::cout.write((char *)pVal, vLen) /* << " " << vLen */;
std::cout << std::endl;
count++;
}
GTEST_ASSERT_EQ(count, nData);
tdbDbcClose(pDBC);
TDB_FREE(pKey);
TDB_FREE(pVal);
}
}
// commit the transaction
tdbCommit(pEnv, &txn);
tdbTxnClose(&txn);
ret = tdbDbDrop(pDb);
GTEST_ASSERT_EQ(ret, 0);
// Close a database
tdbDbClose(pDb);
// Close Env
ret = tdbEnvClose(pEnv);
GTEST_ASSERT_EQ(ret, 0);
}

3
source/util/src/tcompare.c
View File

@ -433,7 +433,8 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) {
char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
int32_t ret = WCSPatternMatch((TdUcs4*)pattern, (TdUcs4*)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo);
int32_t ret =
WCSPatternMatch((TdUcs4 *)pattern, (TdUcs4 *)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo);
taosMemoryFree(pattern);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;

11
source/util/src/tencode.c
View File

@ -33,12 +33,19 @@ void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size,
pCoder->data = data;
pCoder->size = size;
pCoder->pos = 0;
tFreeListInit(&(pCoder->fl));
pCoder->mList = NULL;
TD_SLIST_INIT(&(pCoder->stack));
}
void tCoderClear(SCoder* pCoder) {
tFreeListClear(&(pCoder->fl));
SCoderMem* pMem;
// clear memory
for (pMem = pCoder->mList; pMem; pMem = pCoder->mList) {
pCoder->mList = pMem->next;
taosMemoryFree(pMem);
}
struct SCoderNode* pNode;
for (;;) {
pNode = TD_SLIST_HEAD(&(pCoder->stack));

1
source/util/src/tlog.c
View File

@ -92,6 +92,7 @@ int32_t sDebugFlag = 135;
int32_t tsdbDebugFlag = 131;
int32_t tqDebugFlag = 135;
int32_t fsDebugFlag = 135;
int32_t metaDebugFlag = 135;
int32_t fnDebugFlag = 135;
int64_t dbgEmptyW = 0;

8
source/util/test/encodeTest.cpp
View File

@ -230,7 +230,7 @@ static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) {
const char *tstr;
uint64_t len;
if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1;
TCODER_MALLOC(pSAV1->A_c, char*, len + 1, pCoder);
pSAV1->A_c = (char *)TCODER_MALLOC(pCoder, len + 1);
memcpy(pSAV1->A_c, tstr, len + 1);
tEndDecode(pCoder);
@ -269,7 +269,7 @@ static int32_t tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) {
const char *tstr;
uint64_t len;
if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1;
TCODER_MALLOC(pSAV2->A_c, char*, len + 1, pCoder);
pSAV2->A_c = (char *)TCODER_MALLOC(pCoder, len + 1);
memcpy(pSAV2->A_c, tstr, len + 1);
// ------------------------NEW FIELDS DECODE-------------------------------
@ -305,7 +305,7 @@ static int32_t tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) {
static int32_t tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) {
if (tStartDecode(pCoder) < 0) return -1;
TCODER_MALLOC(ps1->pA, SStructA_v1*, sizeof(*(ps1->pA)), pCoder);
ps1->pA = (SStructA_v1 *)TCODER_MALLOC(pCoder, sizeof(*(ps1->pA)));
if (tSStructA_v1_decode(pCoder, ps1->pA) < 0) return -1;
if (tDecodeI32(pCoder, &ps1->v_a) < 0) return -1;
if (tDecodeI8(pCoder, &ps1->v_b) < 0) return -1;
@ -339,7 +339,7 @@ static int32_t tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) {
static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) {
if (tStartDecode(pCoder) < 0) return -1;
TCODER_MALLOC(ps2->pA, SStructA_v2*, sizeof(*(ps2->pA)), pCoder);
ps2->pA = (SStructA_v2 *)TCODER_MALLOC(pCoder, sizeof(*(ps2->pA)));
if (tSStructA_v2_decode(pCoder, ps2->pA) < 0) return -1;
if (tDecodeI32(pCoder, &ps2->v_a) < 0) return -1;
if (tDecodeI8(pCoder, &ps2->v_b) < 0) return -1;

2
tests/script/jenkins/basic.txt
View File

@ -83,7 +83,7 @@
./test.sh -f tsim/mnode/basic1.sim -m
# --- sma
./test.sh -f tsim/sma/tsmaCreateInsertData.sim
# ./test.sh -f tsim/sma/tsmaCreateInsertData.sim
# --- valgrind
#./test.sh -f tsim/valgrind/checkError.sim -v