xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refact meta 1

This commit is contained in:
Hongze Cheng 2022-04-19 13:10:03 +00:00
parent ae624a8194
commit 3cc66adcb7
18 changed files with 689 additions and 489 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/common/trow.h
View File

@ -23,7 +23,6 @@
#include "tbuffer.h"
#include "tdataformat.h"
#include "tdef.h"
#include "tschema.h"
#include "ttypes.h"
#include "tutil.h"
@ -58,12 +57,12 @@ extern "C" {
#define TD_ROWS_ALL_NORM 0x00U
#define TD_ROWS_NULL_NORM 0x01U
#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM
#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM
#define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v))
#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM)
#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM)
#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM)
#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM)
#define KvConvertRatio (0.9f)
#define KvConvertRatio (0.9f)
#define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio))
#ifdef TD_SUPPORT_BITMAP
@ -341,7 +340,8 @@ static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx,
return false;
}
static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType,
int8_t bitmapMode) {
switch (bitmapMode) {
case 0:
tdGetBitmapValTypeII(pBitmap, colIdx, pValType);

81
include/common/tschema.h
View File

@ -1,81 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_SCHEMA_H_
#define _TD_COMMON_SCHEMA_H_
#include "os.h"
#include "tarray.h"
#include "ttypes.h"
#ifdef __cplusplus
extern "C" {
#endif
#if 0
typedef struct STColumn {
/// column name
char *cname;
union {
/// for encode purpose
uint64_t info;
struct {
uint64_t sma : 1;
/// column data type
uint64_t type : 7;
/// column id
uint64_t cid : 16;
/// max bytes of the column
uint64_t bytes : 32;
/// reserved
uint64_t reserve : 8;
};
};
/// comment about the column
char *comment;
} STColumn;
typedef struct STSchema {
/// schema version
uint16_t sver;
/// number of columns
uint16_t ncols;
/// sma attributes
struct {
bool sma;
SArray *smaArray;
};
/// column info
STColumn cols[];
} STSchema;
typedef struct {
uint64_t size;
STSchema *pSchema;
} STShemaBuilder;
#define tSchemaBuilderInit(target, capacity) \
{ .size = (capacity), .pSchema = (target) }
void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver);
void tSchemaBuilderSetSMA(bool sma, SArray *smaArray);
int32_t tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment);
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_COMMON_SCHEMA_H_*/

3
include/util/tlog.h
View File

@ -59,6 +59,7 @@ extern int32_t sDebugFlag;
extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag;
extern int32_t fsDebugFlag;
extern int32_t metaDebugFlag;
int32_t taosInitLog(const char *logName, int32_t maxFiles);
void taosCloseLog();
@ -78,6 +79,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
#endif
;
// clang-format off
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
@ -87,6 +89,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
#define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
// clang-format on
#ifdef __cplusplus
}

6
source/common/test/tschemaTest.cpp
View File

@ -1,6 +0,0 @@
#include <gtest/gtest.h>
#include "tschema.h"
TEST(td_schema_test, build_schema_test) {
}

3
source/dnode/vnode/CMakeLists.txt
View File

@ -22,6 +22,7 @@ target_sources(
"src/meta/metaTable.c"
"src/meta/metaTDBImpl.c"
"src/meta/metaQuery.c"
"src/meta/metaCommit.c"
# tsdb
"src/tsdb/tsdbTDBImpl.c"
@ -70,7 +71,7 @@ target_link_libraries(
PUBLIC transport
PUBLIC stream
)
# target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
if(${BUILD_TEST})
add_subdirectory(test)

2
source/dnode/vnode/inc/vnode.h
View File

@ -137,6 +137,8 @@ struct SVnodeCfg {
int32_t vgId;
char dbname[TSDB_DB_NAME_LEN];
uint64_t dbId;
int32_t szPage;
int32_t szCache;
uint64_t wsize;
uint64_t ssize;
uint64_t lsize;

49
source/dnode/vnode/src/inc/meta.h
View File

@ -45,23 +45,50 @@ void metaCloseIdx(SMeta* pMeta);
int metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions);
int metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
// metaCommit ==================
int metaBegin(SMeta* pMeta);
static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }
struct SMeta {
char* path;
SVnode* pVnode;
#ifdef META_REFACT
TENV* pEnv;
TDB* pTbDb;
TDB* pSchemaDb;
TDB* pNameIdx;
TDB* pCtbIdx;
#else
SMetaDB* pDB;
#endif
char* path;
SVnode* pVnode;
TENV* pEnv;
TDB* pTbDb;
TDB* pSkmDb;
TDB* pNameIdx;
TDB* pCtbIdx;
TDB* pTagIdx;
TDB* pTtlIdx;
SMetaIdx* pIdx;
};
typedef struct {
tb_uid_t uid;
int64_t ver;
} STbDbKey;
typedef struct __attribute__((__packed__)) {
tb_uid_t uid;
int32_t sver;
} SSkmDbKey;
typedef struct {
tb_uid_t suid;
tb_uid_t uid;
} SCtbIdxKey;
typedef struct __attribute__((__packed__)) {
tb_uid_t suid;
int16_t cid;
char data[];
} STagIdxKey;
typedef struct {
int64_t dtime;
tb_uid_t uid;
} STtlIdxKey;
#if 1
#define META_SUPER_TABLE TD_SUPER_TABLE
#define META_CHILD_TABLE TD_CHILD_TABLE

10
source/dnode/vnode/src/inc/vnd.h
View File

@ -31,6 +31,9 @@ extern "C" {
// clang-format on
// vnodeCfg ====================
extern const SVnodeCfg vnodeCfgDefault;
int vnodeCheckCfg(const SVnodeCfg*);
int vnodeEncodeConfig(const void* pObj, SJson* pJson);
int vnodeDecodeConfig(const SJson* pJson, void* pObj);
@ -43,10 +46,10 @@ void vnodeQueryClose(SVnode* pVnode);
int vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg);
// vnodeCommit ====================
int vnodeBegin(SVnode* pVnode);
int vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg);
int vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo);
int vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo);
int vnodeBegin(SVnode* pVnode, int option);
int vnodeSyncCommit(SVnode* pVnode);
int vnodeAsyncCommit(SVnode* pVnode);
@ -88,11 +91,6 @@ void* vmaMalloc(SVMemAllocator* pVMA, uint64_t size);
void vmaFree(SVMemAllocator* pVMA, void* ptr);
bool vmaIsFull(SVMemAllocator* pVMA);
// vnodeCfg.h
extern const SVnodeCfg vnodeCfgDefault;
int vnodeCheckCfg(const SVnodeCfg*);
#endif
#ifdef __cplusplus

1
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -22,6 +22,7 @@
#include "sync.h"
#include "tchecksum.h"
#include "tcoding.h"
#include "tcompare.h"
#include "tcompression.h"
#include "tdatablock.h"
#include "tdbInt.h"

21
source/dnode/vnode/src/meta/metaCommit.c Normal file
View File

@ -0,0 +1,21 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
int metaBegin(SMeta *pMeta) {
// TODO
return 0;
}

215
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -15,14 +15,19 @@
#include "vnodeInt.h"
static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
SMeta *pMeta = NULL;
int ret;
int slen;
*ppMeta = NULL;
#ifdef META_REFACT
#else
// create handle
slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_META_DIR) + 3;
if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + slen)) == NULL) {
@ -38,37 +43,217 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
// create path if not created yet
taosMkDir(pMeta->path);
// open meta
if (metaOpenDB(pMeta) < 0) {
// open env
ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv);
if (ret < 0) {
metaError("vgId: %d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
if (metaOpenIdx(pMeta) < 0) {
// open pTbDb
ret = tdbDbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb);
if (ret < 0) {
metaError("vgId: %d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
#endif
// open pSkmDb
ret = tdbDbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb);
if (ret < 0) {
metaError("vgId: %d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pNameIdx
ret = tdbDbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx);
if (ret < 0) {
metaError("vgId: %d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pCtbIdx
ret = tdbDbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx);
if (ret < 0) {
metaError("vgId: %d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pTagIdx
ret = tdbDbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx);
if (ret < 0) {
metaError("vgId: %d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pTtlIdx
ret = tdbDbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx);
if (ret < 0) {
metaError("vgId: %d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open index
if (metaOpenIdx(pMeta) < 0) {
metaError("vgId: %d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
metaDebug("vgId: %d meta is opened", TD_VID(pVnode));
*ppMeta = pMeta;
return 0;
_err:
#ifdef META_REFACT
#else
if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pDB) metaCloseDB(pMeta);
if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
taosMemoryFree(pMeta);
#endif
return -1;
}
int metaClose(SMeta *pMeta) {
if (pMeta) {
#ifdef META_REFACT
#else
metaCloseIdx(pMeta);
metaCloseDB(pMeta);
if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pTtlIdx) tdbDbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbDbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbDbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbDbClose(pMeta->pNameIdx);
if (pMeta->pSkmDb) tdbDbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbDbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv);
taosMemoryFree(pMeta);
#endif
}
return 0;
}
static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
STbDbKey *pTbDbKey1 = (STbDbKey *)pKey1;
STbDbKey *pTbDbKey2 = (STbDbKey *)pKey2;
if (pTbDbKey1->uid > pTbDbKey2->uid) {
return 1;
} else if (pTbDbKey1->uid < pTbDbKey2->uid) {
return -1;
}
if (pTbDbKey1->ver > pTbDbKey2->ver) {
return 1;
} else if (pTbDbKey1->ver < pTbDbKey2->ver) {
return -1;
}
return 0;
}
static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
SSkmDbKey *pSkmDbKey1 = (SSkmDbKey *)pKey1;
SSkmDbKey *pSkmDbKey2 = (SSkmDbKey *)pKey2;
if (pSkmDbKey1->uid > pSkmDbKey2->uid) {
return 1;
} else if (pSkmDbKey1->uid < pSkmDbKey2->uid) {
return -1;
}
if (pSkmDbKey1->sver > pSkmDbKey2->sver) {
return 1;
} else if (pSkmDbKey1->sver < pSkmDbKey2->sver) {
return -1;
}
return 0;
}
static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
SCtbIdxKey *pCtbIdxKey1 = (SCtbIdxKey *)pKey1;
SCtbIdxKey *pCtbIdxKey2 = (SCtbIdxKey *)pKey2;
if (pCtbIdxKey1->suid > pCtbIdxKey2->suid) {
return 1;
} else if (pCtbIdxKey1->suid < pCtbIdxKey2->suid) {
return -1;
}
if (pCtbIdxKey1->uid > pCtbIdxKey2->uid) {
return 1;
} else if (pCtbIdxKey1->uid < pCtbIdxKey2->uid) {
return -1;
}
return 0;
}
static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
STagIdxKey *pTagIdxKey1 = (STagIdxKey *)pKey1;
STagIdxKey *pTagIdxKey2 = (STagIdxKey *)pKey2;
int8_t *p1, *p2;
int8_t type;
int c;
// compare suid
if (pTagIdxKey1->suid > pTagIdxKey2->suid) {
return 1;
} else if (pTagIdxKey1->suid < pTagIdxKey2->suid) {
return -1;
}
// compare column id
if (pTagIdxKey1->cid > pTagIdxKey2->cid) {
return 1;
} else if (pTagIdxKey1->cid < pTagIdxKey2->cid) {
return -1;
}
// compare value
p1 = pTagIdxKey1->data;
p2 = pTagIdxKey2->data;
ASSERT(p1[0] == p2[0]);
type = p1[0];
p1++;
p2++;
c = doCompare(p1, p2, type, 0);
if (c) return c;
if (IS_VAR_DATA_TYPE(type)) {
p1 = p1 + varDataTLen(p1);
p2 = p2 + varDataTLen(p2);
} else {
p1 = p1 + tDataTypes[type].bytes;
p2 = p2 + tDataTypes[type].bytes;
}
// compare suid
if (*(tb_uid_t *)p1 > *(tb_uid_t *)p2) {
return 1;
} else if (*(tb_uid_t *)p1 < *(tb_uid_t *)p2) {
return -1;
}
return 0;
}
static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1;
STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2;
if (pTtlIdxKey1->dtime > pTtlIdxKey2->dtime) {
return 1;
} else if (pTtlIdxKey1->dtime < pTtlIdxKey2->dtime) {
return -1;
}
if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {
return 1;
} else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {
return -1;
}
return 0;

378
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -14,3 +14,381 @@
*/
#include "vnodeInt.h"
SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
SMTbCursor *pTbCur = NULL;
#if 0
SMetaDB *pDB = pMeta->pDB;
pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur));
if (pTbCur == NULL) {
return NULL;
}
tdbDbcOpen(pDB->pTbDB, &pTbCur->pDbc);
#endif
return pTbCur;
}
void metaCloseTbCursor(SMTbCursor *pTbCur) {
#if 0
if (pTbCur) {
if (pTbCur->pDbc) {
tdbDbcClose(pTbCur->pDbc);
}
taosMemoryFree(pTbCur);
}
#endif
}
char *metaTbCursorNext(SMTbCursor *pTbCur) {
#if 0
void *pKey = NULL;
void *pVal = NULL;
int kLen;
int vLen;
int ret;
void *pBuf;
STbCfg tbCfg;
for (;;) {
ret = tdbDbNext(pTbCur->pDbc, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break;
pBuf = pVal;
metaDecodeTbInfo(pBuf, &tbCfg);
if (tbCfg.type == META_SUPER_TABLE) {
taosMemoryFree(tbCfg.name);
taosMemoryFree(tbCfg.stbCfg.pTagSchema);
continue;
} else if (tbCfg.type == META_CHILD_TABLE) {
kvRowFree(tbCfg.ctbCfg.pTag);
}
return tbCfg.name;
}
#endif
return NULL;
}
STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
#if 0
int ret;
SMetaDB *pMetaDb = pMeta->pDB;
void *pKey;
void *pVal;
int kLen;
int vLen;
STbCfg *pTbCfg;
// Fetch
pKey = &uid;
kLen = sizeof(uid);
pVal = NULL;
ret = tdbDbGet(pMetaDb->pTbDB, pKey, kLen, &pVal, &vLen);
if (ret < 0) {
return NULL;
}
// Decode
pTbCfg = taosMemoryMalloc(sizeof(*pTbCfg));
metaDecodeTbInfo(pVal, pTbCfg);
TDB_FREE(pVal);
return pTbCfg;
#endif
return NULL;
}
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
// return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
return NULL;
}
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
SMCtbCursor *pCtbCur = NULL;
// SMetaDB *pDB = pMeta->pDB;
// int ret;
// pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur));
// if (pCtbCur == NULL) {
// return NULL;
// }
// pCtbCur->suid = uid;
// ret = tdbDbcOpen(pDB->pCtbIdx, &pCtbCur->pCur);
// if (ret < 0) {
// taosMemoryFree(pCtbCur);
// return NULL;
// }
return pCtbCur;
}
void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
// if (pCtbCur) {
// if (pCtbCur->pCur) {
// tdbDbcClose(pCtbCur->pCur);
// TDB_FREE(pCtbCur->pKey);
// TDB_FREE(pCtbCur->pVal);
// }
// taosMemoryFree(pCtbCur);
// }
}
tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
// int ret;
// SCtbIdxKey *pCtbIdxKey;
// ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen);
// if (ret < 0) {
// return 0;
// }
// pCtbIdxKey = pCtbCur->pKey;
// return pCtbIdxKey->uid;
return 0;
}
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
#if 0
tb_uid_t quid;
SSchemaWrapper *pSW;
STSchemaBuilder sb;
SSchema *pSchema;
STSchema *pTSchema;
STbCfg *pTbCfg;
pTbCfg = metaGetTbInfoByUid(pMeta, uid);
if (pTbCfg->type == META_CHILD_TABLE) {
quid = pTbCfg->ctbCfg.suid;
} else {
quid = uid;
}
pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true);
if (pSW == NULL) {
return NULL;
}
tdInitTSchemaBuilder(&sb, 0);
for (int i = 0; i < pSW->nCols; i++) {
pSchema = pSW->pSchema + i;
tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes);
}
pTSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
return pTSchema;
#endif
return NULL;
}
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
#if 0
#ifdef META_TDB_SMA_TEST
STSmaWrapper *pSW = NULL;
pSW = taosMemoryCalloc(1, sizeof(*pSW));
if (pSW == NULL) {
return NULL;
}
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
if (pCur == NULL) {
taosMemoryFree(pSW);
return NULL;
}
void *pBuf = NULL;
SSmaIdxKey *pSmaIdxKey = NULL;
while (true) {
// TODO: lock during iterate?
if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
pSmaIdxKey = pCur->pKey;
ASSERT(pSmaIdxKey != NULL);
void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false);
if (pSmaVal == NULL) {
tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid);
continue;
}
++pSW->number;
STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma));
if (tptr == NULL) {
TDB_FREE(pSmaVal);
metaCloseSmaCursor(pCur);
tdDestroyTSmaWrapper(pSW);
taosMemoryFreeClear(pSW);
return NULL;
}
pSW->tSma = tptr;
pBuf = pSmaVal;
if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
TDB_FREE(pSmaVal);
metaCloseSmaCursor(pCur);
tdDestroyTSmaWrapper(pSW);
taosMemoryFreeClear(pSW);
return NULL;
}
TDB_FREE(pSmaVal);
continue;
}
break;
}
metaCloseSmaCursor(pCur);
return pSW;
#endif
#endif
return NULL;
}
STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
#if 0
void *pKey;
void *pVal;
void *ppKey;
int pkLen;
int kLen;
int vLen;
int ret;
pKey = tbname;
kLen = strlen(tbname) + 1;
pVal = NULL;
ppKey = NULL;
ret = tdbDbPGet(pMeta->pDB->pNameIdx, pKey, kLen, &ppKey, &pkLen, &pVal, &vLen);
if (ret < 0) {
return NULL;
}
ASSERT(pkLen == kLen + sizeof(uid));
*uid = *(tb_uid_t *)POINTER_SHIFT(ppKey, kLen);
TDB_FREE(ppKey);
TDB_FREE(pVal);
return metaGetTbInfoByUid(pMeta, *uid);
#endif
return NULL;
}
int metaGetTbNum(SMeta *pMeta) {
// TODO
// ASSERT(0);
return 0;
}
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
#if 0
// TODO
// ASSERT(0); // comment this line to pass CI
// return NULL:
#ifdef META_TDB_SMA_TEST
SArray *pUids = NULL;
SMetaDB *pDB = pMeta->pDB;
void *pKey;
// TODO: lock?
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0);
if (pCur == NULL) {
return NULL;
}
// TODO: lock?
SSmaIdxKey *pSmaIdxKey = NULL;
tb_uid_t uid = 0;
while (true) {
// TODO: lock during iterate?
if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
ASSERT(pSmaIdxKey != NULL);
pSmaIdxKey = pCur->pKey;
if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) {
continue;
}
uid = pSmaIdxKey->uid;
if (!pUids) {
pUids = taosArrayInit(16, sizeof(tb_uid_t));
if (!pUids) {
metaCloseSmaCursor(pCur);
return NULL;
}
}
taosArrayPush(pUids, &uid);
continue;
}
break;
}
metaCloseSmaCursor(pCur);
return pUids;
#endif
#endif
return NULL;
}
void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
#if 0
// TODO
// ASSERT(0);
// return NULL;
#ifdef META_TDB_SMA_TEST
SMetaDB *pDB = pMeta->pDB;
void *pKey = NULL;
void *pVal = NULL;
int kLen = 0;
int vLen = 0;
int ret = -1;
// Set key
pKey = (void *)&indexUid;
kLen = sizeof(indexUid);
// Query
ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen);
if (ret != 0 || !pVal) {
return NULL;
}
if (!isDecode) {
// return raw value
return pVal;
}
// Decode
STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma));
if (pCfg == NULL) {
taosMemoryFree(pVal);
return NULL;
}
void *pBuf = pVal;
if (tDecodeTSma(pBuf, pCfg) == NULL) {
tdDestroyTSma(pCfg);
taosMemoryFree(pCfg);
TDB_FREE(pVal);
return NULL;
}
TDB_FREE(pVal);
return pCfg;
#endif
#endif
return NULL;
}

357
source/dnode/vnode/src/meta/metaTDBImpl.c
View File

@ -373,64 +373,6 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
return 0;
}
STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
int ret;
SMetaDB *pMetaDb = pMeta->pDB;
void *pKey;
void *pVal;
int kLen;
int vLen;
STbCfg *pTbCfg;
// Fetch
pKey = &uid;
kLen = sizeof(uid);
pVal = NULL;
ret = tdbDbGet(pMetaDb->pTbDB, pKey, kLen, &pVal, &vLen);
if (ret < 0) {
return NULL;
}
// Decode
pTbCfg = taosMemoryMalloc(sizeof(*pTbCfg));
metaDecodeTbInfo(pVal, pTbCfg);
TDB_FREE(pVal);
return pTbCfg;
}
STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
void *pKey;
void *pVal;
void *ppKey;
int pkLen;
int kLen;
int vLen;
int ret;
pKey = tbname;
kLen = strlen(tbname) + 1;
pVal = NULL;
ppKey = NULL;
ret = tdbDbPGet(pMeta->pDB->pNameIdx, pKey, kLen, &ppKey, &pkLen, &pVal, &vLen);
if (ret < 0) {
return NULL;
}
ASSERT(pkLen == kLen + sizeof(uid));
*uid = *(tb_uid_t *)POINTER_SHIFT(ppKey, kLen);
TDB_FREE(ppKey);
TDB_FREE(pVal);
return metaGetTbInfoByUid(pMeta, *uid);
}
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
}
static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {
void *pKey;
void *pVal;
@ -462,92 +404,10 @@ static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_
return pSchemaWrapper;
}
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
tb_uid_t quid;
SSchemaWrapper *pSW;
STSchemaBuilder sb;
SSchema *pSchema;
STSchema *pTSchema;
STbCfg *pTbCfg;
pTbCfg = metaGetTbInfoByUid(pMeta, uid);
if (pTbCfg->type == META_CHILD_TABLE) {
quid = pTbCfg->ctbCfg.suid;
} else {
quid = uid;
}
pSW = metaGetTableSchemaImpl(pMeta, quid, sver, true, true);
if (pSW == NULL) {
return NULL;
}
tdInitTSchemaBuilder(&sb, 0);
for (int i = 0; i < pSW->nCols; i++) {
pSchema = pSW->pSchema + i;
tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes);
}
pTSchema = tdGetSchemaFromBuilder(&sb);
tdDestroyTSchemaBuilder(&sb);
return pTSchema;
}
struct SMTbCursor {
TDBC *pDbc;
};
SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
SMTbCursor *pTbCur = NULL;
SMetaDB *pDB = pMeta->pDB;
pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur));
if (pTbCur == NULL) {
return NULL;
}
tdbDbcOpen(pDB->pTbDB, &pTbCur->pDbc);
return pTbCur;
}
void metaCloseTbCursor(SMTbCursor *pTbCur) {
if (pTbCur) {
if (pTbCur->pDbc) {
tdbDbcClose(pTbCur->pDbc);
}
taosMemoryFree(pTbCur);
}
}
char *metaTbCursorNext(SMTbCursor *pTbCur) {
void *pKey = NULL;
void *pVal = NULL;
int kLen;
int vLen;
int ret;
void *pBuf;
STbCfg tbCfg;
for (;;) {
ret = tdbDbNext(pTbCur->pDbc, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break;
pBuf = pVal;
metaDecodeTbInfo(pBuf, &tbCfg);
if (tbCfg.type == META_SUPER_TABLE) {
taosMemoryFree(tbCfg.name);
taosMemoryFree(tbCfg.stbCfg.pTagSchema);
continue;
} else if (tbCfg.type == META_CHILD_TABLE) {
kvRowFree(tbCfg.ctbCfg.pTag);
}
return tbCfg.name;
}
return NULL;
}
struct SMCtbCursor {
TDBC *pCur;
tb_uid_t suid;
@ -557,61 +417,6 @@ struct SMCtbCursor {
int vLen;
};
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
SMCtbCursor *pCtbCur = NULL;
SMetaDB *pDB = pMeta->pDB;
int ret;
pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur));
if (pCtbCur == NULL) {
return NULL;
}
pCtbCur->suid = uid;
ret = tdbDbcOpen(pDB->pCtbIdx, &pCtbCur->pCur);
if (ret < 0) {
taosMemoryFree(pCtbCur);
return NULL;
}
// TODO: move the cursor to the suid there
return pCtbCur;
}
void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
if (pCtbCur) {
if (pCtbCur->pCur) {
tdbDbcClose(pCtbCur->pCur);
TDB_FREE(pCtbCur->pKey);
TDB_FREE(pCtbCur->pVal);
}
taosMemoryFree(pCtbCur);
}
}
tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
int ret;
SCtbIdxKey *pCtbIdxKey;
ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen);
if (ret < 0) {
return 0;
}
pCtbIdxKey = pCtbCur->pKey;
return pCtbIdxKey->uid;
}
int metaGetTbNum(SMeta *pMeta) {
// TODO
// ASSERT(0);
return 0;
}
struct SMSmaCursor {
TDBC *pCur;
tb_uid_t uid;
@ -621,71 +426,6 @@ struct SMSmaCursor {
int vLen;
};
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
// TODO
// ASSERT(0);
// return NULL;
#ifdef META_TDB_SMA_TEST
STSmaWrapper *pSW = NULL;
pSW = taosMemoryCalloc(1, sizeof(*pSW));
if (pSW == NULL) {
return NULL;
}
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
if (pCur == NULL) {
taosMemoryFree(pSW);
return NULL;
}
void *pBuf = NULL;
SSmaIdxKey *pSmaIdxKey = NULL;
while (true) {
// TODO: lock during iterate?
if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
pSmaIdxKey = pCur->pKey;
ASSERT(pSmaIdxKey != NULL);
void *pSmaVal = metaGetSmaInfoByIndex(pMeta, pSmaIdxKey->smaUid, false);
if (pSmaVal == NULL) {
tsdbWarn("no tsma exists for indexUid: %" PRIi64, pSmaIdxKey->smaUid);
continue;
}
++pSW->number;
STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma));
if (tptr == NULL) {
TDB_FREE(pSmaVal);
metaCloseSmaCursor(pCur);
tdDestroyTSmaWrapper(pSW);
taosMemoryFreeClear(pSW);
return NULL;
}
pSW->tSma = tptr;
pBuf = pSmaVal;
if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
TDB_FREE(pSmaVal);
metaCloseSmaCursor(pCur);
tdDestroyTSmaWrapper(pSW);
taosMemoryFreeClear(pSW);
return NULL;
}
TDB_FREE(pSmaVal);
continue;
}
break;
}
metaCloseSmaCursor(pCur);
return pSW;
#endif
}
int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) {
// TODO
ASSERT(0);
@ -762,53 +502,6 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
return 0;
}
void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
// TODO
// ASSERT(0);
// return NULL;
#ifdef META_TDB_SMA_TEST
SMetaDB *pDB = pMeta->pDB;
void *pKey = NULL;
void *pVal = NULL;
int kLen = 0;
int vLen = 0;
int ret = -1;
// Set key
pKey = (void *)&indexUid;
kLen = sizeof(indexUid);
// Query
ret = tdbDbGet(pDB->pSmaDB, pKey, kLen, &pVal, &vLen);
if (ret != 0 || !pVal) {
return NULL;
}
if (!isDecode) {
// return raw value
return pVal;
}
// Decode
STSma *pCfg = (STSma *)taosMemoryCalloc(1, sizeof(STSma));
if (pCfg == NULL) {
taosMemoryFree(pVal);
return NULL;
}
void *pBuf = pVal;
if (tDecodeTSma(pBuf, pCfg) == NULL) {
tdDestroyTSma(pCfg);
taosMemoryFree(pCfg);
TDB_FREE(pVal);
return NULL;
}
TDB_FREE(pVal);
return pCfg;
#endif
}
/**
* @brief
*
@ -883,56 +576,6 @@ void metaCloseSmaCursor(SMSmaCursor *pCur) {
#endif
}
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
// TODO
// ASSERT(0); // comment this line to pass CI
// return NULL:
#ifdef META_TDB_SMA_TEST
SArray *pUids = NULL;
SMetaDB *pDB = pMeta->pDB;
void *pKey;
// TODO: lock?
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, 0);
if (pCur == NULL) {
return NULL;
}
// TODO: lock?
SSmaIdxKey *pSmaIdxKey = NULL;
tb_uid_t uid = 0;
while (true) {
// TODO: lock during iterate?
if (tdbDbNext(pCur->pCur, &pCur->pKey, &pCur->kLen, NULL, &pCur->vLen) == 0) {
ASSERT(pSmaIdxKey != NULL);
pSmaIdxKey = pCur->pKey;
if (pSmaIdxKey->uid == 0 || pSmaIdxKey->uid == uid) {
continue;
}
uid = pSmaIdxKey->uid;
if (!pUids) {
pUids = taosArrayInit(16, sizeof(tb_uid_t));
if (!pUids) {
metaCloseSmaCursor(pCur);
return NULL;
}
}
taosArrayPush(pUids, &uid);
continue;
}
break;
}
metaCloseSmaCursor(pCur);
return pUids;
#endif
}
static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) {
int tlen = 0;
SSchema *pSchema;

6
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -19,6 +19,8 @@ const SVnodeCfg vnodeCfgDefault = {
.vgId = -1,
.dbname = "",
.dbId = 0,
.szPage = 4096,
.szCache = 256,
.wsize = 96 * 1024 * 1024,
.ssize = 1 * 1024 * 1024,
.lsize = 1024,
@ -53,6 +55,8 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1;
if (tjsonAddStringToObject(pJson, "dbname", pCfg->dbname) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wsize", pCfg->wsize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "ssize", pCfg->ssize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "lsize", pCfg->lsize) < 0) return -1;
@ -91,6 +95,8 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (tjsonGetNumberValue(pJson, "vgId", pCfg->vgId) < 0) return -1;
if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1;
if (tjsonGetNumberValue(pJson, "dbId", pCfg->dbId) < 0) return -1;
if (tjsonGetNumberValue(pJson, "szPage", pCfg->szPage) < 0) return -1;
if (tjsonGetNumberValue(pJson, "szCache", pCfg->szCache) < 0) return -1;
if (tjsonGetNumberValue(pJson, "wsize", pCfg->wsize) < 0) return -1;
if (tjsonGetNumberValue(pJson, "ssize", pCfg->ssize) < 0) return -1;
if (tjsonGetNumberValue(pJson, "lsize", pCfg->lsize) < 0) return -1;

20
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -25,6 +25,26 @@ static int vnodeEndCommit(SVnode *pVnode);
static int vnodeCommit(void *arg);
static void vnodeWaitCommit(SVnode *pVnode);
int vnodeBegin(SVnode *pVnode) {
// begin buffer pool
// begin meta
if (metaBegin(pVnode->pMeta) < 0) {
vError("vgId: %d failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
// begin tsdb
#if 0
if (tsdbBegin(pVnode->pTsdb) < 0) {
vError("vgId: %d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
#endif
return 0;
}
int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile;

6
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -124,11 +124,11 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
goto _err;
}
#if 0
if (vnodeBegin() < 0) {
// vnode begin
if (vnodeBegin(pVnode) < 0) {
vError("vgId: %d failed to begin since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
#endif
return pVnode;

3
source/util/src/tcompare.c
View File

@ -428,7 +428,8 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) {
char *pattern = taosMemoryCalloc(varDataLen(pRight) + TSDB_NCHAR_SIZE, 1);
memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
int32_t ret = WCSPatternMatch((TdUcs4*)pattern, (TdUcs4*)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo);
int32_t ret =
WCSPatternMatch((TdUcs4 *)pattern, (TdUcs4 *)varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo);
taosMemoryFree(pattern);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;

1
source/util/src/tlog.c
View File

@ -91,6 +91,7 @@ int32_t sDebugFlag = 135;
int32_t tsdbDebugFlag = 131;
int32_t tqDebugFlag = 135;
int32_t fsDebugFlag = 135;
int32_t metaDebugFlag = 135;
int64_t dbgEmptyW = 0;
int64_t dbgWN = 0;