more code

source/dnode/vnode/src/tsdb/dev/tsdbCommit.c

@@ -83,7 +83,7 @@ static int32_t tsdbCommitWriteDelData(SCommitter *pCommitter, int64_t suid, int6
   return code;
 }
 
-static int32_t tsdbCommitTSData(SCommitter *pCommitter) {
+static int32_t commit_timeseries_data(SCommitter *pCommitter) {
   int32_t code = 0;
   int32_t lino;
 
@@ -126,7 +126,7 @@ _exit:
   return code;
 }
 
-static int32_t tsdbCommitDLData(SCommitter *pCommitter) {
+static int32_t commit_delete_data(SCommitter *pCommitter) {
   int32_t code = 0;
   int32_t lino;
 
@@ -165,7 +165,7 @@ _exit:
   return code;
 }
 
-static int32_t tsdbCommitFSetStart(SCommitter *pCommitter) {
+static int32_t start_commit_file_set(SCommitter *pCommitter) {
   pCommitter->fid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision);
   tsdbFidKeyRange(pCommitter->fid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey,
                   &pCommitter->maxKey);
@@ -178,7 +178,7 @@ static int32_t tsdbCommitFSetStart(SCommitter *pCommitter) {
   return 0;
 }
 
-static int32_t tsdbCommitFSetEnd(SCommitter *pCommitter) {
+static int32_t end_commit_file_set(SCommitter *pCommitter) {
   int32_t code = 0;
   int32_t lino = 0;
 
@@ -191,23 +191,23 @@ _exit:
   return code;
 }
 
-static int32_t tsdbCommitNextFSet(SCommitter *pCommitter) {
+static int32_t commit_next_file_set(SCommitter *pCommitter) {
   int32_t code = 0;
   int32_t lino = 0;
 
   // fset commit start
-  code = tsdbCommitFSetStart(pCommitter);
+  code = start_commit_file_set(pCommitter);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // commit fset
-  code = tsdbCommitTSData(pCommitter);
+  code = commit_timeseries_data(pCommitter);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code = tsdbCommitDLData(pCommitter);
+  code = commit_delete_data(pCommitter);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // fset commit end
-  code = tsdbCommitFSetEnd(pCommitter);
+  code = end_commit_file_set(pCommitter);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -218,7 +218,7 @@ _exit:
   return code;
 }
 
-static int32_t tsdbCommitterOpen(STsdb *pTsdb, SCommitInfo *pInfo, SCommitter *pCommitter) {
+static int32_t open_committer(STsdb *pTsdb, SCommitInfo *pInfo, SCommitter *pCommitter) {
   int32_t code = 0;
   int32_t lino;
 
@@ -250,7 +250,7 @@ _exit:
   return code;
 }
 
-static int32_t tsdbCommitterClose(SCommitter *pCommiter, int32_t eno) {
+static int32_t close_committer(SCommitter *pCommiter, int32_t eno) {
   int32_t code = 0;
   // TODO
   return code;
@@ -280,15 +280,15 @@ int32_t tsdbCommitBegin(STsdb *pTsdb, SCommitInfo *pInfo) {
   } else {
     SCommitter committer;
 
-    code = tsdbCommitterOpen(pTsdb, pInfo, &committer);
+    code = open_committer(pTsdb, pInfo, &committer);
     TSDB_CHECK_CODE(code, lino, _exit);
 
     while (committer.nextKey != TSKEY_MAX) {
-      code = tsdbCommitNextFSet(&committer);
+      code = commit_next_file_set(&committer);
       if (code) break;
     }
 
-    code = tsdbCommitterClose(&committer, code);
+    code = close_committer(&committer, code);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 

source/dnode/vnode/src/tsdb/dev/tsdbFS.c

@@ -14,3 +14,12 @@
  */
 
 #include "dev.h"
+
+struct STFileSystem {
+  STsdb *pTsdb;
+};
+
+int32_t tsdbOpenFileSystem(STsdb *pTsdb, struct STFileSystem **ppFS) {
+  // TODO
+  return 0;
+}

source/dnode/vnode/src/tsdb/dev/tsdbFSet.c

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "dev.h"
+
+struct SFileSet {
+  struct STFile *files[TSDB_FTYPE_MAX];
+  SRBTree        fsttTree;
+};

source/dnode/vnode/src/tsdb/dev/tsdbFSet.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TSDB_FILE_SET_H
+#define _TSDB_FILE_SET_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Exposed Handle */
+struct SFileSet;
+
+/* Exposed APIs */
+
+/* Exposed Structs */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TSDB_FILE_SET_H*/

source/dnode/vnode/src/tsdb/dev/tsdbFile.h

@@ -23,40 +23,25 @@ extern "C" {
 #endif
 
 /* Exposed Handle */
-typedef struct STFile  STFile;
+struct STFile;
-typedef struct SFileOp SFileOp;
+struct SFileOp;
 
 typedef enum {
-  TSDB_FTYPE_NONE = 0,  // no file type
+  TSDB_FTYPE_HEAD = 0,  // .head
-  TSDB_FTYPE_STT,       // .stt
-  TSDB_FTYPE_HEAD,      // .head
   TSDB_FTYPE_DATA,      // .data
   TSDB_FTYPE_SMA,       // .sma
   TSDB_FTYPE_TOMB,      // .tomb
+  TSDB_FTYPE_MAX,       // max file type
+  TSDB_FTYPE_STT,       // .stt
 } tsdb_ftype_t;
 
 /* Exposed APIs */
 
 /* Exposed Structs */
-typedef struct SFStt {
+struct FStt {
-  int64_t offset;
+  int32_t level;
-} SFStt;
+  int32_t nStt;
-
+};
-typedef struct SFHead {
-  int64_t offset;
-} SFHead;
-
-typedef struct SFData {
-  // TODO
-} SFData;
-
-typedef struct SFSma {
-  // TODO
-} SFSma;
-
-typedef struct SFTomb {
-  // TODO
-} SFTomb;
 struct STFile {
   char         fname[TSDB_FILENAME_LEN];
   SDiskID      diskId;
@@ -65,13 +50,6 @@ struct STFile {
   int32_t      fid;
   int32_t      ref;
   tsdb_ftype_t type;
-  union {
-    SFStt  fstt;
-    SFHead fhead;
-    SFData fdata;
-    SFSma  fsma;
-    SFTomb ftomb;
-  };
 };
 
 #ifdef __cplusplus

source/dnode/vnode/src/tsdb/dev/tsdbSttFWriter.c

@@ -32,10 +32,12 @@ struct SSttFWriter {
   // helper data
   SSkmInfo skmTb;
   SSkmInfo skmRow;
+  int32_t  aBufSize[5];
+  uint8_t *aBuf[5];
   STsdbFD *pFd;
 };
 
-static int32_t write_ts_block(struct SSttFWriter *pWriter) {
+static int32_t write_timeseries_block(struct SSttFWriter *pWriter) {
   int32_t code = 0;
   int32_t lino;
 
@@ -50,37 +52,67 @@ static int32_t write_ts_block(struct SSttFWriter *pWriter) {
   pSttBlk->minUid = pBData->aUid[0];
   pSttBlk->maxUid = pBData->aUid[pBData->nRow - 1];
   pSttBlk->minKey = pSttBlk->maxKey = pBData->aTSKEY[0];
-  pSttBlk->minVer = pSttBlk->maxVer = pBData->aTSKEY[0];
+  pSttBlk->minVer = pSttBlk->maxVer = pBData->aVersion[0];
   pSttBlk->nRow = pBData->nRow;
   for (int32_t iRow = 1; iRow < pBData->nRow; iRow++) {
-    pSttBlk->minKey = TMIN(pSttBlk->minKey, pBData->aTSKEY[iRow]);
+    if (pSttBlk->minKey > pBData->aTSKEY[iRow]) pSttBlk->minKey = pBData->aTSKEY[iRow];
-    pSttBlk->maxKey = TMAX(pSttBlk->maxKey, pBData->aTSKEY[iRow]);
+    if (pSttBlk->maxKey < pBData->aTSKEY[iRow]) pSttBlk->maxKey = pBData->aTSKEY[iRow];
-    pSttBlk->minVer = TMIN(pSttBlk->minVer, pBData->aVersion[iRow]);
+    if (pSttBlk->minVer > pBData->aVersion[iRow]) pSttBlk->minVer = pBData->aVersion[iRow];
-    pSttBlk->maxVer = TMAX(pSttBlk->maxVer, pBData->aVersion[iRow]);
+    if (pSttBlk->maxVer < pBData->aVersion[iRow]) pSttBlk->maxVer = pBData->aVersion[iRow];
   }
 
   // compress data block
-  code = tCmprBlockData(pBData, pWriter->config.cmprAlg, NULL, NULL, NULL /* TODO */, NULL /* TODO */);
+  code = tCmprBlockData(pBData, pWriter->config.cmprAlg, NULL, NULL, pWriter->config.aBuf, pWriter->aBufSize);
   TSDB_CHECK_CODE(code, lino, _exit);
 
+  pSttBlk->bInfo.offset = pWriter->config.file.size;
+  pSttBlk->bInfo.szKey = pWriter->aBufSize[2] + pWriter->aBufSize[3];
+  pSttBlk->bInfo.szBlock = pWriter->aBufSize[0] + pWriter->aBufSize[1] + pSttBlk->bInfo.szKey;
+
+  for (int32_t iBuf = 3; iBuf >= 0; iBuf--) {
+    if (pWriter->aBufSize[iBuf]) {
+      code =
+          tsdbWriteFile(pWriter->pFd, pWriter->config.file.size, pWriter->config.aBuf[iBuf], pWriter->aBufSize[iBuf]);
+      TSDB_CHECK_CODE(code, lino, _exit);
+
+      pWriter->config.file.size += pWriter->aBufSize[iBuf];
+    }
+  }
+
   tBlockDataClear(pBData);
 
 _exit:
   if (code) {
     tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->config.pTsdb->pVnode), __func__, lino,
               tstrerror(code));
+  } else {
+    // tsdbTrace();
+  }
+  return code;
+}
+
+static int32_t write_stt_blk(struct SSttFWriter *pWriter) {
+  int32_t code = 0;
+  int32_t lino;
+
+  if (taosArrayGetSize(pWriter->aSttBlk) == 0) {
+    goto _exit;
+  }
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->config.pTsdb->pVnode), __func__, lino,
+              tstrerror(code));
+  } else {
+    // tsdbDebug("vgId:%d %s done, offset:%" PRId64 " size:%" PRId64 " # of stt block:%d",
+    // TD_VID(pWriter->config.pTsdb->pVnode), __func__);
   }
   return code;
 }
 
 static int32_t write_del_block(struct SSttFWriter *pWriter) {
   int32_t code = 0;
-  // TODO
-  return code;
-}
 
-static int32_t write_stt_blk(struct SSttFWriter *pWriter) {
-  int32_t code = 0;
   // TODO
   return code;
 }
@@ -91,7 +123,7 @@ static int32_t write_del_blk(struct SSttFWriter *pWriter) {
   return code;
 }
 
-static int32_t stt_fwriter_create(const struct SSttFWriterConf *pConf, struct SSttFWriter **ppWriter) {
+static int32_t create_stt_fwriter(const struct SSttFWriterConf *pConf, struct SSttFWriter **ppWriter) {
   int32_t code = 0;
 
   if ((ppWriter[0] = taosMemoryCalloc(1, sizeof(*ppWriter[0]))) == NULL) {
@@ -100,11 +132,14 @@ static int32_t stt_fwriter_create(const struct SSttFWriterConf *pConf, struct SS
   }
 
   ppWriter[0]->config = pConf[0];
+  if (pConf->pSkmTb == NULL) {
+    ppWriter[0]->config.pSkmTb = &ppWriter[0]->skmTb;
+  }
   if (pConf->pSkmRow == NULL) {
     ppWriter[0]->config.pSkmRow = &ppWriter[0]->skmRow;
   }
-  if (pConf->pSkmTb == NULL) {
+  if (pConf->aBuf == NULL) {
-    ppWriter[0]->config.pSkmTb = &ppWriter[0]->skmTb;
+    ppWriter[0]->config.aBuf = ppWriter[0]->aBuf;
   }
 
   tBlockDataCreate(&ppWriter[0]->bData);
@@ -124,10 +159,11 @@ _exit:
   return code;
 }
 
-static int32_t stt_fwriter_destroy(struct SSttFWriter *pWriter) {
+static int32_t destroy_stt_fwriter(struct SSttFWriter *pWriter) {
   if (pWriter) {
     tDestroyTSchema(pWriter->skmTb.pTSchema);
     tDestroyTSchema(pWriter->skmRow.pTSchema);
+    for (int32_t i = 0; i < sizeof(pWriter->aBuf) / sizeof(pWriter->aBuf[0]); i++) taosMemoryFree(pWriter->aBuf[i]);
     taosArrayDestroy(pWriter->aSttBlk);
     tBlockDataDestroy(&pWriter->bData);
     taosMemoryFree(pWriter);
@@ -135,26 +171,36 @@ static int32_t stt_fwriter_destroy(struct SSttFWriter *pWriter) {
   return 0;
 }
 
-static int32_t stt_fwriter_open(struct SSttFWriter *pWriter) {
+static int32_t open_stt_fwriter(struct SSttFWriter *pWriter) {
   int32_t code = 0;
-  // TODO
+  int32_t lino;
+
+  int32_t flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;  // TODO
+
+  code = tsdbOpenFile(pWriter->config.file.fname, pWriter->config.szPage, flag, &pWriter->pFd);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->config.pTsdb->pVnode), __func__, lino,
+              tstrerror(code));
+  }
   return code;
 }
 
-static int32_t stt_fwriter_close(struct SSttFWriter *pWriter) {
+static int32_t close_stt_fwriter(struct SSttFWriter *pWriter) {
-  int32_t code = 0;
+  tsdbCloseFile(&pWriter->pFd);
-  // TODO
+  return 0;
-  return code;
 }
 
 int32_t tsdbSttFWriterOpen(const struct SSttFWriterConf *pConf, struct SSttFWriter **ppWriter) {
   int32_t code = 0;
   int32_t lino;
 
-  code = stt_fwriter_create(pConf, ppWriter);
+  code = create_stt_fwriter(pConf, ppWriter);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  code = stt_fwriter_open(ppWriter[0]);
+  code = open_stt_fwriter(ppWriter[0]);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -173,10 +219,22 @@ int32_t tsdbSttFWriterClose(struct SSttFWriter **ppWriter) {
   int32_t code = 0;
   int32_t lino;
 
-  code = stt_fwriter_close(ppWriter[0]);
+  if (ppWriter[0]->bData.nRow > 0) {
+    code = write_timeseries_block(ppWriter[0]);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  code = write_stt_blk(ppWriter[0]);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  stt_fwriter_close(ppWriter[0]);
+  code = write_del_blk(ppWriter[0]);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  code = close_stt_fwriter(ppWriter[0]);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  destroy_stt_fwriter(ppWriter[0]);
+  ppWriter[0] = NULL;
 
 _exit:
   if (code) {
@@ -191,21 +249,20 @@ int32_t tsdbSttFWriteTSData(struct SSttFWriter *pWriter, TABLEID *tbid, TSDBROW
 
   if (!TABLE_SAME_SCHEMA(pWriter->bData.suid, pWriter->bData.uid, tbid->suid, tbid->uid)) {
     if (pWriter->bData.nRow > 0) {
-      code = write_ts_block(pWriter);
+      code = write_timeseries_block(pWriter);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
 
-    // TODO: code = tsdbUpdateTableSchema(pWriter->config.pTsdb, tbid->uid, tbid->suid, pWriter->config.pSkmTb);
+    code = tsdbUpdateSkmTb(pWriter->config.pTsdb, tbid, pWriter->config.pSkmTb);
     TSDB_CHECK_CODE(code, lino, _exit);
 
-    TABLEID id = {.suid = tbid->suid, .uid = tbid->suid ? 0 : tbid->uid};
+    TABLEID id = {.suid = tbid->suid, 0};
     code = tBlockDataInit(&pWriter->bData, &id, pWriter->config.pSkmTb->pTSchema, NULL, 0);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
   if (pRow->type == TSDBROW_ROW_FMT) {
-    // TODO: code = tsdbUpdateRowSchema(pWriter->config.pTsdb, tbid->uid, tbid->suid, pRow->row,
+    code = tsdbUpdateSkmRow(pWriter->config.pTsdb, tbid, TSDBROW_SVERSION(pRow), pWriter->config.pSkmRow);
-    // pWriter->config.pSkmRow);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
@@ -213,7 +270,7 @@ int32_t tsdbSttFWriteTSData(struct SSttFWriter *pWriter, TABLEID *tbid, TSDBROW
   TSDB_CHECK_CODE(code, lino, _exit);
 
   if (pWriter->bData.nRow >= pWriter->config.maxRow) {
-    code = write_ts_block(pWriter);
+    code = write_timeseries_block(pWriter);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 

source/dnode/vnode/src/tsdb/dev/tsdbSttFWriter.h

@@ -32,12 +32,13 @@ int32_t tsdbSttFWriteDLData(struct SSttFWriter *pWriter, TABLEID *tbid, SDelData
 
 struct SSttFWriterConf {
   STsdb        *pTsdb;
+  struct STFile file;
   SSkmInfo     *pSkmTb;
   SSkmInfo     *pSkmRow;
-  STFile    file;
   int32_t       maxRow;
   int32_t       szPage;
   int8_t        cmprAlg;
+  uint8_t     **aBuf;
 };
 
 #ifdef __cplusplus

source/dnode/vnode/src/tsdb/dev/tsdbUtil.c

@@ -54,3 +54,34 @@ int32_t tDelBlockAppend(SDelBlock *pDelBlock, const TABLEID *tbid, const SDelDat
 
   return code;
 }
+
+int32_t tsdbUpdateSkmTb(STsdb *pTsdb, const TABLEID *tbid, SSkmInfo *pSkmTb) {
+  if (tbid->suid) {
+    if (pSkmTb->suid == tbid->suid) {
+      pSkmTb->uid = tbid->uid;
+      return 0;
+    }
+  } else if (pSkmTb->uid == tbid->uid) {
+    return 0;
+  }
+
+  pSkmTb->suid = tbid->suid;
+  pSkmTb->uid = tbid->uid;
+  tDestroyTSchema(pSkmTb->pTSchema);
+  return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, tbid->suid, tbid->uid, -1, &pSkmTb->pTSchema);
+}
+
+int32_t tsdbUpdateSkmRow(STsdb *pTsdb, const TABLEID *tbid, int32_t sver, SSkmInfo *pSkmRow) {
+  if (pSkmRow->pTSchema && pSkmRow->suid == tbid->suid) {
+    if (pSkmRow->suid) {
+      if (sver == pSkmRow->pTSchema->version) return 0;
+    } else if (pSkmRow->uid == tbid->uid && pSkmRow->pTSchema->version == sver) {
+      return 0;
+    }
+  }
+
+  pSkmRow->suid = tbid->suid;
+  pSkmRow->uid = tbid->uid;
+  tDestroyTSchema(pSkmRow->pTSchema);
+  return metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, tbid->suid, tbid->uid, sver, &pSkmRow->pTSchema);
+}

source/dnode/vnode/src/tsdb/dev/tsdbUtil.h

@@ -32,6 +32,9 @@ int32_t tDelBlockDestroy(SDelBlock *pDelBlock);
 int32_t tDelBlockClear(SDelBlock *pDelBlock);
 int32_t tDelBlockAppend(SDelBlock *pDelBlock, const TABLEID *tbid, const SDelData *pDelData);
 
+int32_t tsdbUpdateSkmTb(STsdb *pTsdb, const TABLEID *tbid, SSkmInfo *pSkmTb);
+int32_t tsdbUpdateSkmRow(STsdb *pTsdb, const TABLEID *tbid, int32_t sver, SSkmInfo *pSkmRow);
+
 /* Exposed Structs */
 struct SDelBlock {
   SColData aColData[5];  // <suid, uid, version, skey, ekey>