xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

use unit test to test speed

This commit is contained in:
Hongze Cheng 2020-06-23 07:33:31 +00:00
parent 5ecda58f48
commit ab64145d4b
3 changed files with 55 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/tsdb/src/tsdbMain.c
View File

@ -179,7 +179,7 @@ int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *
return -1; return -1;
} }
} }
pRsp->affectedRows = htonl(affectedrows); if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows);
return 0; return 0;
} }
@ -648,7 +648,7 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) {
} }
pRepo->config = *pCfg; pRepo->config = *pCfg;
pRepo->appH = *pAppH; if (pAppH) pRepo->appH = *pAppH;
pRepo->tsdbMeta = tsdbNewMeta(pCfg); pRepo->tsdbMeta = tsdbNewMeta(pCfg);
if (pRepo->tsdbMeta == NULL) { if (pRepo->tsdbMeta == NULL) {

2
src/tsdb/tests/CMakeLists.txt
View File

@ -1,6 +1,6 @@
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
add_executable(tsdbTests ${SOURCE_LIST}) add_executable(tsdbTests ${SOURCE_LIST})
target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb) target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb tutil trpc)
add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests) add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests)

251
src/tsdb/tests/tsdbTests.cpp
View File

@ -2,9 +2,8 @@
#include <stdlib.h> #include <stdlib.h>
#include <sys/time.h> #include <sys/time.h>
#include "tdataformat.h" #include "tsdb.h"
#include "tsdbMain.h" #include "tsdbMain.h"
#include "tskiplist.h"
static double getCurTime() { static double getCurTime() {
struct timeval tv; struct timeval tv;
@ -77,7 +76,7 @@ static int insertData(SInsertInfo *pInfo) {
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
pMsg->compressed = htonl(pMsg->numOfBlocks); pMsg->compressed = htonl(pMsg->numOfBlocks);
if (tsdbInsertData(pInfo->pRepo, pMsg) < 0) { if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) {
tfree(pMsg); tfree(pMsg);
return -1; return -1;
} }
@ -90,222 +89,72 @@ static int insertData(SInsertInfo *pInfo) {
return 0; return 0;
} }
TEST(TsdbTest, DISABLED_tableEncodeDecode) { static void tsdbSetCfg(STsdbCfg *pCfg, int32_t tsdbId, int32_t cacheBlockSize, int32_t totalBlocks, int32_t maxTables,
// TEST(TsdbTest, tableEncodeDecode) { int32_t daysPerFile, int32_t keep, int32_t minRows, int32_t maxRows, int8_t precision,
STable *pTable = (STable *)malloc(sizeof(STable)); int8_t compression) {
pCfg->tsdbId = tsdbId;
pTable->type = TSDB_NORMAL_TABLE; pCfg->cacheBlockSize = cacheBlockSize;
pTable->tableId.uid = 987607499877672L; pCfg->totalBlocks = totalBlocks;
pTable->tableId.tid = 0; pCfg->maxTables = maxTables;
pTable->superUid = -1; pCfg->daysPerFile = daysPerFile;
pTable->sversion = 0; pCfg->keep = keep;
pTable->tagSchema = NULL; pCfg->minRowsPerFileBlock = minRows;
pTable->tagVal = NULL; pCfg->maxRowsPerFileBlock = maxRows;
int nCols = 5; pCfg->precision = precision;
STSchema *schema = tdNewSchema(nCols); pCfg->compression = compression;
for (int i = 0; i < nCols; i++) {
if (i == 0) {
tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
} else {
tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
}
}
pTable->schema = schema;
int bufLen = 0;
void *buf = tsdbEncodeTable(pTable, &bufLen);
STable *tTable = tsdbDecodeTable(buf, bufLen);
ASSERT_EQ(pTable->type, tTable->type);
ASSERT_EQ(pTable->tableId.uid, tTable->tableId.uid);
ASSERT_EQ(pTable->tableId.tid, tTable->tableId.tid);
ASSERT_EQ(pTable->superUid, tTable->superUid);
ASSERT_EQ(pTable->sversion, tTable->sversion);
ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0);
} }
// TEST(TsdbTest, DISABLED_createRepo) { static void tsdbSetTableCfg(STableCfg *pCfg) {
TEST(TsdbTest, createRepo) { STSchemaBuilder schemaBuilder = {0};
STsdbCfg config;
STsdbRepo *repo;
// 1. Create a tsdb repository pCfg->type = TSDB_NORMAL_TABLE;
tsdbSetDefaultCfg(&config); pCfg->superUid = TSDB_INVALID_SUPER_TABLE_ID;
ASSERT_EQ(tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL), 0); pCfg->tableId.tid = 1;
pCfg->tableId.uid = 5849583783847394;
tdInitTSchemaBuilder(&schemaBuilder, 0);
TSDB_REPO_T *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); int colId = 0;
ASSERT_NE(pRepo, nullptr); for (int i = 0; i < 5; i++) {
tdAddColToSchema(&schemaBuilder, (colId == 0) ? TSDB_DATA_TYPE_TIMESTAMP : TSDB_DATA_TYPE_INT, colId, 0);
// 2. Create a normal table colId++;
STableCfg tCfg;
ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
tsdbTableSetName(&tCfg, "test", false);
int nCols = 5;
STSchema *schema = tdNewSchema(nCols);
for (int i = 0; i < nCols; i++) {
if (i == 0) {
tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
} else {
tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
}
} }
tsdbTableSetSchema(&tCfg, schema, true); pCfg->schema = tdGetSchemaFromBuilder(&schemaBuilder);
pCfg->name = strdup("t1");
tsdbCreateTable(pRepo, &tCfg); tdDestroyTSchemaBuilder(&schemaBuilder);
// Insert Some Data
SInsertInfo iInfo = {
.pRepo = pRepo,
// .isAscend = true,
.isAscend = false,
.tid = tCfg.tableId.tid,
.uid = tCfg.tableId.uid,
.sversion = tCfg.sversion,
.startTime = 1584081000000,
.interval = 1000,
.totalRows = 10000000,
.rowsPerSubmit = 1,
.pSchema = schema
};
ASSERT_EQ(insertData(&iInfo), 0);
// Close the repository
tsdbCloseRepo(pRepo);
// Open the repository again
pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
repo = (STsdbRepo *)pRepo;
ASSERT_NE(pRepo, nullptr);
// // Insert more data
// iInfo.startTime = iInfo.startTime + iInfo.interval * iInfo.totalRows;
// iInfo.totalRows = 10;
// iInfo.pRepo = pRepo;
// ASSERT_EQ(insertData(&iInfo), 0);
// // Close the repository
// tsdbCloseRepo(pRepo);
// // Open the repository again
// pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
// repo = (STsdbRepo *)pRepo;
// ASSERT_NE(pRepo, nullptr);
// // Read from file
// SRWHelper rhelper;
// tsdbInitReadHelper(&rhelper, repo);
// SFileGroup *pFGroup = tsdbSearchFGroup(repo->tsdbFileH, 1833);
// ASSERT_NE(pFGroup, nullptr);
// ASSERT_GE(tsdbSetAndOpenHelperFile(&rhelper, pFGroup), 0);
// STable *pTable = tsdbGetTableByUid(repo->tsdbMeta, tCfg.tableId.uid);
// ASSERT_NE(pTable, nullptr);
// tsdbSetHelperTable(&rhelper, pTable, repo);
// ASSERT_EQ(tsdbLoadCompInfo(&rhelper, NULL), 0);
// ASSERT_EQ(tsdbLoadBlockData(&rhelper, blockAtIdx(&rhelper, 0), NULL), 0);
int k = 0;
} }
TEST(TsdbTest, DISABLED_openRepo) { TEST(TsdbTest, testInsertSpeed) {
// TEST(TsdbTest, openRepo) { int vnode = 1;
// tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL); int ret = 0;
// ASSERT_NE(repo, nullptr); STsdbCfg tsdbCfg;
STableCfg tableCfg;
std::string testDir = "/root/mnt/work/ttest";
char * rootDir = strdup((testDir + "/vnode" + std::to_string(vnode)).c_str());
// STsdbRepo *pRepo = (STsdbRepo *)repo; tsdbDebugFlag = 131; //NOTE: you must set the flag
// SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1655); taosRemoveDir(rootDir);
// for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { // Create and open repository
// tsdbOpenFile(&pGroup->files[type], O_RDONLY); tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1);
// } tsdbCreateRepo(rootDir, &tsdbCfg);
TSDB_REPO_T *repo = tsdbOpenRepo(rootDir, NULL);
ASSERT_NE(repo, nullptr);
// SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx)); // Create table
// tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables); tsdbSetTableCfg(&tableCfg);
tsdbCreateTable(repo, &tableCfg);
// SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len); // Insert data
SInsertInfo iInfo = {repo, true, 1, 5849583783847394, 0, 1590000000000, 10, 10000000, 1000, tableCfg.schema};
// tsdbLoadCompBlocks(pGroup, &pIdx[1], (void *)pCompInfo); insertData(&iInfo);
// int blockIdx = 0; tsdbCloseRepo(repo, 1);
// SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]);
// SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
// tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
// STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
// SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5);
// tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
// tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);
// tdResetDataCols(pDataCols);
// tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock + 1, 1, pDataCols, pCompData);
// int k = 0;
}
TEST(TsdbTest, DISABLED_createFileGroup) {
SFileGroup fGroup;
// ASSERT_EQ(tsdbCreateFileGroup("/home/ubuntu/work/ttest/vnode0/data", 1820, &fGroup, 1000), 0);
int k = 0;
} }
static char *getTKey(const void *data) { static char *getTKey(const void *data) {
return (char *)data; return (char *)data;
} }
static void insertSkipList(bool isAscend) {
TSKEY start_time = 1587393453000;
TSKEY interval = 1000;
SSkipList *pList = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(TSKEY), 0, 0, 1, getTKey);
ASSERT_NE(pList, nullptr);
for (size_t i = 0; i < 20000000; i++)
{
TSKEY time = isAscend ? (start_time + i * interval) : (start_time - i * interval);
int32_t level = 0;
int32_t headSize = 0;
tSkipListNewNodeInfo(pList, &level, &headSize);
SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(TSKEY));
ASSERT_NE(pNode, nullptr);
pNode->level = level;
*(TSKEY *)((char *)pNode + headSize) = time;
tSkipListPut(pList, pNode);
}
tSkipListDestroy(pList);
}
TEST(TsdbTest, DISABLED_testSkipList) {
// TEST(TsdbTest, testSkipList) {
double stime = getCurTime();
insertSkipList(true);
double etime = getCurTime();
printf("Time used to insert 100000000 records takes %f seconds\n", etime-stime);
stime = getCurTime();
insertSkipList(false);
etime = getCurTime();
printf("Time used to insert 100000000 records takes %f seconds\n", etime-stime);
}