xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(tsdb): unit of days and keep support minute

This commit is contained in:
Cary Xu 2022-05-12 18:32:32 +08:00
parent 8d2754259d
commit d66bb65171
8 changed files with 25 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/common/tglobal.h
View File

@ -43,7 +43,7 @@ extern int32_t tsMaxNumOfDistinctResults;
extern int32_t tsCompatibleModel; extern int32_t tsCompatibleModel;
extern bool tsEnableSlaveQuery; extern bool tsEnableSlaveQuery;
extern bool tsPrintAuth; extern bool tsPrintAuth;
extern int64_t tsTickPerDay[3]; extern int64_t tsTickPerMin[3];
// multi-process // multi-process
extern bool tsMultiProcess; extern bool tsMultiProcess;
@ -122,7 +122,7 @@ extern int32_t tsDiskCfgNum;
extern SDiskCfg tsDiskCfg[]; extern SDiskCfg tsDiskCfg[];
// udf // udf
extern bool tsStartUdfd; extern bool tsStartUdfd;
// internal // internal
extern int32_t tsTransPullupInterval; extern int32_t tsTransPullupInterval;

10
source/common/src/tglobal.c
View File

@ -153,11 +153,11 @@ bool tsStreamSchedV = true;
/* /*
* minimum scale for whole system, millisecond by default * minimum scale for whole system, millisecond by default
* for TSDB_TIME_PRECISION_MILLI: 86400000L * for TSDB_TIME_PRECISION_MILLI: 60000L
* TSDB_TIME_PRECISION_MICRO: 86400000000L * TSDB_TIME_PRECISION_MICRO: 60000000L
* TSDB_TIME_PRECISION_NANO: 86400000000000L * TSDB_TIME_PRECISION_NANO: 60000000000L
*/ */
int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L};
// lossy compress 6 // lossy compress 6
char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty
@ -170,7 +170,7 @@ uint32_t tsCurRange = 100; // range
char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
// udf // udf
bool tsStartUdfd = true; bool tsStartUdfd = true;
// internal // internal
int32_t tsTransPullupInterval = 6; int32_t tsTransPullupInterval = 6;

8
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -145,10 +145,10 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->szBuf = pCreate->buffer * 1024 * 1024; pCfg->szBuf = pCreate->buffer * 1024 * 1024;
pCfg->isWeak = true; pCfg->isWeak = true;
pCfg->tsdbCfg.precision = pCreate->precision; pCfg->tsdbCfg.precision = pCreate->precision;
pCfg->tsdbCfg.days = 10; pCfg->tsdbCfg.days = pCreate->daysPerFile;
pCfg->tsdbCfg.keep0 = 3650; pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0;
pCfg->tsdbCfg.keep1 = 3650; pCfg->tsdbCfg.keep1 = pCreate->daysToKeep1;
pCfg->tsdbCfg.keep2 = 3650; pCfg->tsdbCfg.keep2 = pCreate->daysToKeep2;
pCfg->tsdbCfg.minRows = pCreate->minRows; pCfg->tsdbCfg.minRows = pCreate->minRows;
pCfg->tsdbCfg.maxRows = pCreate->maxRows; pCfg->tsdbCfg.maxRows = pCreate->maxRows;
for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) { for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) {

8
source/dnode/vnode/src/inc/tsdb.h
View File

@ -518,9 +518,9 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
if (key < 0) { if (key < 0) {
return (int)((key + 1) / tsTickPerDay[precision] / days - 1); return (int)((key + 1) / tsTickPerMin[precision] / days - 1);
} else { } else {
return (int)((key / tsTickPerDay[precision] / days)); return (int)((key / tsTickPerMin[precision] / days));
} }
} }
@ -770,8 +770,8 @@ static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) {
} }
static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) { static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) {
*minKey = fid * days * tsTickPerDay[precision]; *minKey = fid * days * tsTickPerMin[precision];
*maxKey = *minKey + days * tsTickPerDay[precision] - 1; *maxKey = *minKey + days * tsTickPerMin[precision] - 1;
} }
static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet *pSet) { static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet *pSet) {

9
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -216,9 +216,9 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
TSKEY minKey, midKey, maxKey, now; TSKEY minKey, midKey, maxKey, now;
now = taosGetTimestamp(pCfg->precision); now = taosGetTimestamp(pCfg->precision);
minKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; minKey = now - pCfg->keep2 * tsTickPerMin[pCfg->precision];
midKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision]; midKey = now - pCfg->keep1 * tsTickPerMin[pCfg->precision];
maxKey = now - pCfg->keep0 * tsTickPerDay[pCfg->precision]; maxKey = now - pCfg->keep0 * tsTickPerMin[pCfg->precision];
pRtn->minKey = minKey; pRtn->minKey = minKey;
pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->days, pCfg->precision)); pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->days, pCfg->precision));
@ -398,7 +398,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
++mIter; ++mIter;
} else if (pIter && !pIter->pTable) { } else if (pIter && !pIter->pTable) {
// When table already dropped during commit, pIter is not NULL but pIter->pTable is NULL. // When table already dropped during commit, pIter is not NULL but pIter->pTable is NULL.
++mIter; // skip the table and do nothing ++mIter; // skip the table and do nothing
} else if (pIdx) { } else if (pIdx) {
if (tsdbMoveBlkIdx(pCommith, pIdx) < 0) { if (tsdbMoveBlkIdx(pCommith, pIdx) < 0) {
tsdbCloseCommitFile(pCommith, true); tsdbCloseCommitFile(pCommith, true);
@ -948,7 +948,6 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) {
static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) {
STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
pCommith->pTable = pTable; pCommith->pTable = pTable;
if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) {

6
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -323,7 +323,7 @@ static int64_t getEarliestValidTimestamp(STsdb* pTsdb) {
STsdbKeepCfg* pCfg = REPO_KEEP_CFG(pTsdb); STsdbKeepCfg* pCfg = REPO_KEEP_CFG(pTsdb);
int64_t now = taosGetTimestamp(pCfg->precision); int64_t now = taosGetTimestamp(pCfg->precision);
return now - (tsTickPerDay[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick return now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick
} }
static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableDataCond* pCond) { static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableDataCond* pCond) {
@ -1047,10 +1047,10 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio
} }
if (key < 0) { if (key < 0) {
key -= (daysPerFile * tsTickPerDay[precision]); key -= (daysPerFile * tsTickPerMin[precision]);
} }
int64_t fid = (int64_t)(key / (daysPerFile * tsTickPerDay[precision])); // set the starting fileId int64_t fid = (int64_t)(key / (daysPerFile * tsTickPerMin[precision])); // set the starting fileId
if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32 if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32
fid = INT32_MIN; fid = INT32_MIN;
} }

2
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -1017,7 +1017,7 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe
int32_t daysPerFile = pCfg->days; int32_t daysPerFile = pCfg->days;
if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { if (storageLevel == SMA_STORAGE_LEVEL_TSDB) {
int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerDay[pCfg->precision]); int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]);
daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS; daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS;
} }

4
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -63,8 +63,8 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
STSRow *row = NULL; STSRow *row = NULL;
STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pTsdb); STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pTsdb);
TSKEY now = taosGetTimestamp(pCfg->precision); TSKEY now = taosGetTimestamp(pCfg->precision);
TSKEY minKey = now - tsTickPerDay[pCfg->precision] * pCfg->keep2; TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
TSKEY maxKey = now + tsTickPerDay[pCfg->precision] * pCfg->days; TSKEY maxKey = now + tsTickPerMin[pCfg->precision] * pCfg->days;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
// pMsg->length = htonl(pMsg->length); // pMsg->length = htonl(pMsg->length);