Merge branch '2.0' of https://github.com/taosdata/TDengine into 2.0

src/inc/trpc.h

@@ -33,83 +33,42 @@ extern "C" {
 #define TAOS_SOCKET_TYPE_NAME_TCP  "tcp"
 #define TAOS_SOCKET_TYPE_NAME_UDP  "udp"
 
-#define TAOS_ID_ASSIGNED   0
-#define TAOS_ID_FREE       1
-#define TAOS_ID_REALLOCATE 2
-
 #define TAOS_CONN_SOCKET_TYPE_S()  ((strcasecmp(tsSocketType, TAOS_SOCKET_TYPE_NAME_UDP) == 0)? TAOS_CONN_UDPS:TAOS_CONN_TCPS)
 #define TAOS_CONN_SOCKET_TYPE_C()  ((strcasecmp(tsSocketType, TAOS_SOCKET_TYPE_NAME_UDP) == 0)? TAOS_CONN_UDPC:TAOS_CONN_TCPC)
 
-#define taosSendMsgToPeer(x, y, z) taosSendMsgToPeerH(x, y, z, NULL)
+extern int tsRpcHeadSize;
-#define taosOpenRpcChann(x, y, z) taosOpenRpcChannWithQ(x,y,z,NULL)
-#define taosBuildReqMsg(x, y) taosBuildReqMsgWithSize(x, y, 512)
-#define taosBuildRspMsg(x, y) taosBuildRspMsgWithSize(x, y, 512)
 
 typedef struct {
   char *localIp;                        // local IP used
   uint16_t localPort;                   // local port
   char *label;                          // for debug purpose
   int   numOfThreads;                   // number of threads to handle connections
-  void *(*fp)(char *, void *, void *);  // function to process the incoming msg
+  void *(*fp)(char type, char *pCont, int contLen, void *handle, int index);  // function to process the incoming msg
-  void *qhandle;                        // queue handle
+  int   sessions;                       // number of sessions allowed
-  int   bits;                           // number of bits for sessionId
-  int   numOfChanns;                    // number of channels
-  int   sessionsPerChann;               // number of sessions per channel
-  int   idMgmt;                         // TAOS_ID_ASSIGNED, TAOS_ID_FREE
   int   connType;                       // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
   int   idleTime;                       // milliseconds, 0 means idle timer is disabled
-  int   noFree;                         // not free buffer
+  char *meterId;   // meter ID
-  void (*efp)(int cid);                 // call back function to process not activated chann
+  char  spi;       // security parameter index
-  int (*afp)(char *meterId, char *spi, char *encrypt, uint8_t *secret,
+  char  encrypt;   // encrypt algorithm
-             uint8_t *ckey);  // call back to retrieve auth info
+  char *secret;    // key for authentication
+  char *ckey;      // ciphering key
+  int   (*afp) (char *meterId, char *spi, char *encrypt, uint8_t *secret, uint8_t *ckey); // call back to retrieve auth info
 } SRpcInit;
 
 typedef struct {
-  int      cid;       // channel ID
+  int16_t   index; 
-  int      sid;       // session ID
+  int16_t   numOfIps;
-  char *   meterId;   // meter ID
+  uint32_t  ip[TSDB_MAX_REPLICA];
-  uint32_t peerId;    // peer link ID
+} SRpcIpSet;
-  void *   shandle;   // pointer returned by taosOpenRpc
-  void *   ahandle;   // handle provided by app
-  char *   peerIp;    // peer IP string
-  uint16_t    peerPort;  // peer port
-  char     spi;       // security parameter index
-  char     encrypt;   // encrypt algorithm
-  char *   secret;    // key for authentication
-  char *   ckey;      // ciphering key
-} SRpcConnInit;
 
-extern int tsRpcHeadSize;
+void *rpcOpen(SRpcInit *pRpc);
+void  rpcClose(void *);
+char *rpcMallocCont(int contLen);
+void  rpcFreeCont(char *pCont);
+void  rpcSendRequest(void *thandle, SRpcIpSet ipSet, char msgType, void *pCont, int contLen, void *ahandle);
+void  rpcSendResponse(void *pConn, void *pCont, int contLen);
+void  rpcSendSimpleRsp(void *pConn, int32_t code);
 
-void *taosOpenRpc(SRpcInit *pRpc);
-
-void taosCloseRpc(void *);
-
-int taosOpenRpcChannWithQ(void *handle, int cid, int sessions, void *qhandle);
-
-void taosCloseRpcChann(void *handle, int cid);
-
-void *taosOpenRpcConn(SRpcConnInit *pInit, uint8_t *code);
-
-void taosCloseRpcConn(void *thandle);
-
-void taosStopRpcConn(void *thandle);
-
-int taosSendMsgToPeerH(void *thandle, char *pCont, int contLen, void *ahandle);
-
-char *taosBuildReqHeader(void *param, char type, char *msg);
-
-char *taosBuildReqMsgWithSize(void *, char type, int size);
-
-char *taosBuildRspMsgWithSize(void *, char type, int size);
-
-int taosSendSimpleRsp(void *thandle, char rsptype, char code);
-
-int taosSetSecurityInfo(int cid, int sid, char *id, int spi, int encrypt, char *secret, char *ckey);
-
-void taosGetRpcConnInfo(void *thandle, uint32_t *peerId, uint32_t *peerIp, uint16_t *peerPort, int *cid, int *sid);
-
-int taosGetOutType(void *thandle);
 
 #ifdef __cplusplus
 }

src/rpc/inc/tcache.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TSCCACHE_H
+#define TDENGINE_TSCCACHE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void *taosOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, int64_t keepTimer);
+
+void taosCloseConnCache(void *handle);
+
+void *taosAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, char *user);
+
+void *taosGetConnFromCache(void *handle, uint32_t ip, uint16_t port, char *user);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TSCACHE_H

src/rpc/src/tcache.c

@@ -0,0 +1,264 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+
+#include "tglobalcfg.h"
+#include "tlog.h"
+#include "tmempool.h"
+#include "tsclient.h"
+#include "ttime.h"
+#include "ttimer.h"
+#include "tutil.h"
+
+typedef struct _c_hash_t {
+  uint32_t          ip;
+  uint16_t          port;
+  struct _c_hash_t *prev;
+  struct _c_hash_t *next;
+  void *            data;
+  uint64_t          time;
+} SConnHash;
+
+typedef struct {
+  SConnHash **    connHashList;
+  mpool_h         connHashMemPool;
+  int             maxSessions;
+  int             total;
+  int *           count;
+  int64_t         keepTimer;
+  pthread_mutex_t mutex;
+  void (*cleanFp)(void *);
+  void *tmrCtrl;
+  void *pTimer;
+} SConnCache;
+
+int taosHashConn(void *handle, uint32_t ip, uint16_t port, char *user) {
+  SConnCache *pObj = (SConnCache *)handle;
+  int         hash = 0;
+  // size_t    user_len = strlen(user);
+
+  hash = ip >> 16;
+  hash += (unsigned short)(ip & 0xFFFF);
+  hash += port;
+  while (*user != '\0') {
+    hash += *user;
+    user++;
+  }
+
+  hash = hash % pObj->maxSessions;
+
+  return hash;
+}
+
+void taosRemoveExpiredNodes(SConnCache *pObj, SConnHash *pNode, int hash, uint64_t time) {
+  if (pNode == NULL) return;
+  if (time < pObj->keepTimer + pNode->time) return;
+
+  SConnHash *pPrev = pNode->prev, *pNext;
+
+  while (pNode) {
+    (*pObj->cleanFp)(pNode->data);
+    pNext = pNode->next;
+    pObj->total--;
+    pObj->count[hash]--;
+    tscTrace("%p ip:0x%x:%hu:%d:%p removed, connections in cache:%d", pNode->data, pNode->ip, pNode->port, hash, pNode,
+             pObj->count[hash]);
+    taosMemPoolFree(pObj->connHashMemPool, (char *)pNode);
+    pNode = pNext;
+  }
+
+  if (pPrev)
+    pPrev->next = NULL;
+  else
+    pObj->connHashList[hash] = NULL;
+}
+
+void *taosAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, char *user) {
+  int         hash;
+  SConnHash * pNode;
+  SConnCache *pObj;
+
+  uint64_t time = taosGetTimestampMs();
+
+  pObj = (SConnCache *)handle;
+  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
+
+  if (data == NULL) {
+    tscTrace("data:%p ip:%p:%d not valid, not added in cache", data, ip, port);
+    return NULL;
+  }
+
+  hash = taosHashConn(pObj, ip, port, user);
+  pNode = (SConnHash *)taosMemPoolMalloc(pObj->connHashMemPool);
+  pNode->ip = ip;
+  pNode->port = port;
+  pNode->data = data;
+  pNode->prev = NULL;
+  pNode->time = time;
+
+  pthread_mutex_lock(&pObj->mutex);
+
+  pNode->next = pObj->connHashList[hash];
+  if (pObj->connHashList[hash] != NULL) (pObj->connHashList[hash])->prev = pNode;
+  pObj->connHashList[hash] = pNode;
+
+  pObj->total++;
+  pObj->count[hash]++;
+  taosRemoveExpiredNodes(pObj, pNode->next, hash, time);
+
+  pthread_mutex_unlock(&pObj->mutex);
+
+  tscTrace("%p ip:0x%x:%hu:%d:%p added, connections in cache:%d", data, ip, port, hash, pNode, pObj->count[hash]);
+
+  return pObj;
+}
+
+void taosCleanConnCache(void *handle, void *tmrId) {
+  int         hash;
+  SConnHash * pNode;
+  SConnCache *pObj;
+
+  pObj = (SConnCache *)handle;
+  if (pObj == NULL || pObj->maxSessions == 0) return;
+  if (pObj->pTimer != tmrId) return;
+
+  uint64_t time = taosGetTimestampMs();
+
+  for (hash = 0; hash < pObj->maxSessions; ++hash) {
+    pthread_mutex_lock(&pObj->mutex);
+    pNode = pObj->connHashList[hash];
+    taosRemoveExpiredNodes(pObj, pNode, hash, time);
+    pthread_mutex_unlock(&pObj->mutex);
+  }
+
+  // tscTrace("timer, total connections in cache:%d", pObj->total);
+  taosTmrReset(taosCleanConnCache, pObj->keepTimer * 2, pObj, pObj->tmrCtrl, &pObj->pTimer);
+}
+
+void *taosGetConnFromCache(void *handle, uint32_t ip, uint16_t port, char *user) {
+  int         hash;
+  SConnHash * pNode;
+  SConnCache *pObj;
+  void *      pData = NULL;
+
+  pObj = (SConnCache *)handle;
+  if (pObj == NULL || pObj->maxSessions == 0) return NULL;
+
+  uint64_t time = taosGetTimestampMs();
+
+  hash = taosHashConn(pObj, ip, port, user);
+  pthread_mutex_lock(&pObj->mutex);
+
+  pNode = pObj->connHashList[hash];
+  while (pNode) {
+    if (time >= pObj->keepTimer + pNode->time) {
+      taosRemoveExpiredNodes(pObj, pNode, hash, time);
+      pNode = NULL;
+      break;
+    }
+
+    if (pNode->ip == ip && pNode->port == port) break;
+
+    pNode = pNode->next;
+  }
+
+  if (pNode) {
+    taosRemoveExpiredNodes(pObj, pNode->next, hash, time);
+
+    if (pNode->prev) {
+      pNode->prev->next = pNode->next;
+    } else {
+      pObj->connHashList[hash] = pNode->next;
+    }
+
+    if (pNode->next) {
+      pNode->next->prev = pNode->prev;
+    }
+
+    pData = pNode->data;
+    taosMemPoolFree(pObj->connHashMemPool, (char *)pNode);
+    pObj->total--;
+    pObj->count[hash]--;
+  }
+
+  pthread_mutex_unlock(&pObj->mutex);
+
+  if (pData) {
+    tscTrace("%p ip:0x%x:%hu:%d:%p retrieved, connections in cache:%d", pData, ip, port, hash, pNode, pObj->count[hash]);
+  }
+
+  return pData;
+}
+
+void *taosOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, int64_t keepTimer) {
+  SConnHash **connHashList;
+  mpool_h     connHashMemPool;
+  SConnCache *pObj;
+
+  connHashMemPool = taosMemPoolInit(maxSessions, sizeof(SConnHash));
+  if (connHashMemPool == 0) return NULL;
+
+  connHashList = calloc(sizeof(SConnHash *), maxSessions);
+  if (connHashList == 0) {
+    taosMemPoolCleanUp(connHashMemPool);
+    return NULL;
+  }
+
+  pObj = malloc(sizeof(SConnCache));
+  if (pObj == NULL) {
+    taosMemPoolCleanUp(connHashMemPool);
+    free(connHashList);
+    return NULL;
+  }
+  memset(pObj, 0, sizeof(SConnCache));
+
+  pObj->count = calloc(sizeof(int), maxSessions);
+  pObj->total = 0;
+  pObj->keepTimer = keepTimer;
+  pObj->maxSessions = maxSessions;
+  pObj->connHashMemPool = connHashMemPool;
+  pObj->connHashList = connHashList;
+  pObj->cleanFp = cleanFp;
+  pObj->tmrCtrl = tmrCtrl;
+  taosTmrReset(taosCleanConnCache, pObj->keepTimer * 2, pObj, pObj->tmrCtrl, &pObj->pTimer);
+
+  pthread_mutex_init(&pObj->mutex, NULL);
+
+  return pObj;
+}
+
+void taosCloseConnCache(void *handle) {
+  SConnCache *pObj;
+
+  pObj = (SConnCache *)handle;
+  if (pObj == NULL || pObj->maxSessions == 0) return;
+
+  pthread_mutex_lock(&pObj->mutex);
+
+  taosTmrStopA(&(pObj->pTimer));
+
+  if (pObj->connHashMemPool) taosMemPoolCleanUp(pObj->connHashMemPool);
+
+  tfree(pObj->connHashList);
+  tfree(pObj->count)
+
+      pthread_mutex_unlock(&pObj->mutex);
+
+  pthread_mutex_destroy(&pObj->mutex);
+
+  memset(pObj, 0, sizeof(SConnCache));
+  free(pObj);
+}

src/util/inc/tbuffer.h

@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2020 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <setjmp.h>
+
+#ifndef TDENGINE_TBUFFER_H
+#define TDENGINE_TBUFFER_H
+
+
+/*
+SBuffer can be used to read or write a buffer, but cannot be used for both
+read & write at a same time. Below is an example:
+
+int main(int argc, char** argv) {
+  //--------------------- write ------------------------
+  SBuffer wbuf;
+  int32_t code = tbufBeginWrite(&wbuf);
+  if (code != 0) {
+    // handle errors
+    return 0;
+  }
+
+  // reserve 1024 bytes for the buffer to improve performance
+  tbufEnsureCapacity(&wbuf, 1024);
+
+  // write 5 integers to the buffer
+  for (int i = 0; i < 5; i++) {
+    tbufWriteInt32(&wbuf, i);
+  }
+
+  // write a string to the buffer
+  tbufWriteString(&wbuf, "this is a string.\n");
+
+  // acquire the result and close the write buffer
+  size_t size = tbufTell(&wbuf);
+  char*  data = tbufGetData(&wbuf, true);
+  tbufClose(&wbuf, true);
+
+
+  //------------------------ read -----------------------
+  SBuffer rbuf;
+  code = tbufBeginRead(&rbuf, data, size);
+  if (code != 0) {
+    printf("you will see this message after print out 5 integers and a string.\n");
+    tbufClose(&rbuf, false);
+    return 0;
+  }
+
+  // read & print out 5 integers
+  for (int i = 0; i < 5; i++) {
+    printf("%d\n", tbufReadInt32(&rbuf));
+  }
+
+  // read & print out a string
+  printf(tbufReadString(&rbuf, NULL));
+
+  // try read another integer, this result in an error as there no this integer
+  tbufReadInt32(&rbuf);
+
+  printf("you should not see this message.\n");
+  tbufClose(&rbuf, false);
+
+  return 0;
+}
+*/
+typedef struct {
+  jmp_buf jb;
+  char* data;
+  size_t pos;
+  size_t size;
+} SBuffer;
+
+
+// common functions can be used in both read & write
+#define tbufThrowError(buf, code) longjmp((buf)->jb, (code))
+size_t tbufTell(SBuffer* buf);
+size_t tbufSeekTo(SBuffer* buf, size_t pos);
+size_t tbufSkip(SBuffer* buf, size_t size);
+void tbufClose(SBuffer* buf, bool keepData);
+
+
+// basic read functions
+#define tbufBeginRead(buf, data, len) (((buf)->data = (char*)data), ((buf)->pos = 0), ((buf)->size = ((data) == NULL) ? 0 : (len)), setjmp((buf)->jb))
+char* tbufRead(SBuffer* buf, size_t size);
+void tbufReadToBuffer(SBuffer* buf, void* dst, size_t size);
+const char* tbufReadString(SBuffer* buf, size_t* len);
+size_t tbufReadToString(SBuffer* buf, char* dst, size_t size);
+
+
+// basic write functions
+#define tbufBeginWrite(buf) ((buf)->data = NULL, ((buf)->pos = 0), ((buf)->size = 0), setjmp((buf)->jb))
+void tbufEnsureCapacity(SBuffer* buf, size_t size);
+char* tbufGetData(SBuffer* buf, bool takeOver);
+void tbufWrite(SBuffer* buf, const void* data, size_t size);
+void tbufWriteAt(SBuffer* buf, size_t pos, const void* data, size_t size);
+void tbufWriteStringLen(SBuffer* buf, const char* str, size_t len);
+void tbufWriteString(SBuffer* buf, const char* str);
+
+
+// read & write function for primitive types
+#ifndef TBUFFER_DEFINE_FUNCTION
+#define TBUFFER_DEFINE_FUNCTION(type, name) \
+  type tbufRead##name(SBuffer* buf); \
+  void tbufWrite##name(SBuffer* buf, type data); \
+  void tbufWrite##name##At(SBuffer* buf, size_t pos, type data);
+#endif
+
+TBUFFER_DEFINE_FUNCTION( bool, Bool )
+TBUFFER_DEFINE_FUNCTION( char, Char )
+TBUFFER_DEFINE_FUNCTION( int8_t, Int8 )
+TBUFFER_DEFINE_FUNCTION( uint8_t, Unt8 )
+TBUFFER_DEFINE_FUNCTION( int16_t, Int16 )
+TBUFFER_DEFINE_FUNCTION( uint16_t, Uint16 )
+TBUFFER_DEFINE_FUNCTION( int32_t, Int32 )
+TBUFFER_DEFINE_FUNCTION( uint32_t, Uint32 )
+TBUFFER_DEFINE_FUNCTION( int64_t, Int64 )
+TBUFFER_DEFINE_FUNCTION( uint64_t, Uint64 )
+TBUFFER_DEFINE_FUNCTION( float, Float )
+TBUFFER_DEFINE_FUNCTION( double, Double )
+
+#endif

src/util/src/tbuffer.c

@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2020 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdlib.h>
+#include <memory.h>
+#include <assert.h>
+
+#define TBUFFER_DEFINE_FUNCTION(type, name) \
+  type tbufRead##name(SBuffer* buf) {            \
+    type ret;                                    \
+    tbufReadToBuffer(buf, &ret, sizeof(type));   \
+    return ret;                                  \
+  }\
+  void tbufWrite##name(SBuffer* buf, type data) {\
+    tbufWrite(buf, &data, sizeof(data));\
+  }\
+  void tbufWrite##name##At(SBuffer* buf, size_t pos, type data) {\
+    tbufWriteAt(buf, pos, &data, sizeof(data));\
+  }
+
+#include "../inc/tbuffer.h"
+
+
+////////////////////////////////////////////////////////////////////////////////
+// common functions
+
+size_t tbufTell(SBuffer* buf) {
+  return buf->pos;
+}
+
+size_t tbufSeekTo(SBuffer* buf, size_t pos) {
+  if (pos > buf->size) {
+    // TODO: update error code, other tbufThrowError need to be changed too
+    tbufThrowError(buf, 1);
+  }
+  size_t old = buf->pos;
+  buf->pos = pos;
+  return old;
+}
+
+size_t tbufSkip(SBuffer* buf, size_t size) {
+  return tbufSeekTo(buf, buf->pos + size);
+}
+
+void tbufClose(SBuffer* buf, bool keepData) {
+  if (!keepData) {
+    free(buf->data);
+  }
+  buf->data = NULL;
+  buf->pos = 0;
+  buf->size = 0;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// read functions
+
+char* tbufRead(SBuffer* buf, size_t size) {
+  char* ret = buf->data + buf->pos;
+  tbufSkip(buf, size);
+  return ret;
+}
+
+void tbufReadToBuffer(SBuffer* buf, void* dst, size_t size) {
+  assert(dst != NULL);
+  // always using memcpy, leave optimization to compiler
+  memcpy(dst, tbufRead(buf, size), size);
+}
+
+const char* tbufReadString(SBuffer* buf, size_t* len) {
+  uint16_t l = tbufReadUint16(buf);
+  char*    ret = buf->data + buf->pos;
+  tbufSkip(buf, l + 1);
+  ret[l] = 0;  // ensure the string end with '\0'
+  if (len != NULL) {
+    *len = l;
+  }
+  return ret;
+}
+
+size_t tbufReadToString(SBuffer* buf, char* dst, size_t size) {
+  assert(dst != NULL);
+  size_t      len;
+  const char* str = tbufReadString(buf, &len);
+  if (len >= size) {
+    len = size - 1;
+  }
+  memcpy(dst, str, len);
+  dst[len] = 0;
+  return len;
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+// write functions
+
+void tbufEnsureCapacity(SBuffer* buf, size_t size) {
+  size += buf->pos;
+  if (size > buf->size) {
+    size_t nsize = size + buf->size;
+    char* data = realloc(buf->data, nsize);
+    if (data == NULL) {
+      tbufThrowError(buf, 2);
+    }
+    buf->data = data;
+    buf->size = nsize;
+  }
+}
+
+char* tbufGetData(SBuffer* buf, bool takeOver) {
+		char* ret = buf->data;
+		if (takeOver) {
+      buf->pos = 0;
+      buf->size = 0;
+			buf->data = NULL;
+    }
+		return ret;
+}
+
+void tbufEndWrite(SBuffer* buf) {
+    free(buf->data);
+    buf->data = NULL;
+    buf->pos = 0;
+    buf->size = 0;
+}
+
+void tbufWrite(SBuffer* buf, const void* data, size_t size) {
+  assert(data != NULL);
+  tbufEnsureCapacity(buf, size);
+  memcpy(buf->data + buf->pos, data, size);
+  buf->pos += size;
+}
+
+void tbufWriteAt(SBuffer* buf, size_t pos, const void* data, size_t size) {
+  assert(data != NULL);
+  // this function can only be called to fill the gap on previous writes,
+  // so 'pos + size <= buf->pos' must be true
+  assert(pos + size <= buf->pos);
+  memcpy(buf->data + pos, data, size);
+}
+
+void tbufWriteStringLen(SBuffer* buf, const char* str, size_t len) {
+  // maximum string length is 65535, if longer string is required
+  // this function and the corresponding read function need to be
+  // revised.
+  assert(len <= 0xffff);
+  tbufWriteUint16(buf, (uint16_t)len);
+  tbufWrite(buf, str, len + 1);
+}
+
+void tbufWriteString(SBuffer* buf, const char* str) {
+  tbufWriteStringLen(buf, str, strlen(str));
+}

src/vnode/CMakeLists.txt

@@ -1,4 +1,7 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+cmake_minimum_required(VERSION 2.8)
-PROJECT(TDengine)
 
-ADD_SUBDIRECTORY(detail)
+project(tsdb)
+
+add_subdirectory(common)
+
+add_subdirectory(tsdb)

src/vnode/common/CMakeLists.txt

@@ -0,0 +1,8 @@
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
+
+list(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/src/vnodePeer.c)
+
+message(STATUS "Common source file ${SOURCE_LIST}")
+
+add_library(common ${SOURCE_LIST})
+target_include_directories(common PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc")

src/vnode/common/src/schema.c

@@ -51,7 +51,7 @@ SISchema tdConvertSchemaToInline(SSchema *pSchema) {
   char *pName = TD_ISCHEMA_COL_NAMES(pISchema);
   for (int32_t i = 0; i < totalCols; i++) {
     SColumn *pCol = TD_SCHEMA_COLUMN_AT(TD_ISCHEMA_SCHEMA(pISchema), i);
-    char *   colName = TD_COLUMN_NAME(TD_SCHEMA_COLUMN_AT(pSchema, i), i);
+    char *   colName = TD_COLUMN_NAME(TD_SCHEMA_COLUMN_AT(pSchema, i));
 
     TD_COLUMN_NAME(pCol) = pName;
 

src/vnode/tsdb/CMakeLists.txt

@@ -0,0 +1,9 @@
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST)
+
+message(STATUS "tsdb source files: ${SOURCE_LIST}")
+
+add_library(tsdb STATIC ${SOURCE_LIST})
+
+target_link_libraries(tsdb common)
+
+target_include_directories(tsdb PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc")

src/vnode/tsdb/inc/tsdb.h

@@ -8,7 +8,7 @@
 #include <stdbool.h>
 #include <stdint.h>
 
-#include "cache.h"
+// #include "cache.h"
 #include "schema.h"
 
 #define TSDB_VERSION_MAJOR 1
@@ -18,6 +18,21 @@ typedef void    tsdb_repo_t;  // use void to hide implementation details from ou
 typedef int32_t table_id_t;   // table ID type in this repository
 typedef int16_t tsdb_id_t;    // TSDB repository ID
 
+// Submit message
+typedef struct {
+  int32_t numOfTables;
+  char    data[];
+} SSubmitMsg;
+
+// Submit message for one table
+typedef struct {
+  table_id_t tableId;       // table ID to insert
+  int32_t    sversion;      // data schema version
+  int32_t    numOfRows;     // number of rows data
+  int64_t    uid;           // table UID to insert
+  char       data[];
+} SSubmitBlock;
+
 // Retention policy.
 typedef struct {
   // TODO: Need a more fancy description
@@ -54,7 +69,7 @@ typedef struct {
   SDataShardPolicy dataShardPolicy;
   SBlockRowsPolicy blockRowsPolicy;
   SRetentionPolicy retentionPlicy;  // retention configuration
-  SCachePool *     cachePool;       // the cache pool the repository to use
+  void *           cachePool;       // the cache pool the repository to use
 } STSDBCfg;
 
 // the TSDB repository info
@@ -205,6 +220,9 @@ typedef struct STimeWindow {
   int64_t ekey;
 } STimeWindow;
 
+typedef struct {
+} SColumnFilterInfo;
+
 // query condition to build vnode iterator
 typedef struct STSDBQueryCond {
   STimeWindow       twindow;
@@ -237,6 +255,10 @@ typedef struct STableIDList {
   int32_t   num;
 } STableIDList;
 
+typedef struct {
+
+} SFields;
+
 /**
  * Get the data block iterator, starting from position according to the query condition
  * @param pRepo  the TSDB repository to query on

src/vnode/tsdb/inc/tsdbCache.h

@@ -3,7 +3,7 @@
 
 #include <stdint.h>
 
-#include "cache.h"
+// #include "cache.h"
 
 #define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16*1024*1024 /* 16M */
 
@@ -13,11 +13,11 @@ typedef struct {
   int32_t numOfRows // numOfRows
 } STableCacheInfo;
 
-typedef struct {
+typedef struct _tsdb_cache_block {
-  char *pData;
+  char *                    pData;
-  STableCacheInfo *pTableInfo;
+  STableCacheInfo *         pTableInfo;
-  SCacheBlock *prev;
+  struct _tsdb_cache_block *prev;
-  SCacheBlock *next;
+  struct _tsdb_cache_block *next;
 } STSDBCacheBlock;
 
 // Use a doublely linked list to implement this
@@ -28,7 +28,6 @@ typedef struct STSDBCache {
   void *           current;
 } SCacheHandle;
 
-
 // ---- Operation on STSDBCacheBlock
 #define TSDB_CACHE_BLOCK_DATA(pBlock) ((pBlock)->pData)
 #define TSDB_CACHE_AVAIL_SPACE(pBlock) ((char *)((pBlock)->pTableInfo) - ((pBlock)->pData))

src/vnode/tsdb/inc/tsdbFile.h

@@ -1,7 +1,8 @@
 #if !defined(_TD_TSDB_FILE_H_)
 #define _TD_TSDB_FILE_H_
 
-#include "tstring.h"
+#include <stdint.h>
+// #include "tstring.h"
 
 typedef int32_t file_id_t;
 
@@ -24,7 +25,7 @@ typedef struct {
 } SFileInfo;
 
 typedef struct {
-  tstring_t fname;
+  char *    fname;
   SFileInfo fInfo;
 } SFILE;
 

src/vnode/tsdb/inc/tsdbMeta.h

@@ -4,7 +4,7 @@
 
 #include <pthread.h>
 
-#include "taosdef.h"
+// #include "taosdef.h"
 
 // Initially, there are 4 tables
 #define TSDB_INIT_NUMBER_OF_SUPER_TABLE 4
@@ -30,7 +30,7 @@ typedef struct STable {
   // For TSDB_SUPER_TABLE, it is the schema including tags
   // For TSDB_NTABLE, it is only the schema, not including tags
   // For TSDB_STABLE, it is NULL
-  SVSchema *pSchema;
+  SSchema *pSchema;
 
   // Tag value for this table
   // For TSDB_SUPER_TABLE and TSDB_NTABLE, it is NULL
@@ -75,7 +75,7 @@ typedef struct {
 #define TSDB_TABLE_CACHE_DATA(pTable) ((pTable)->content.pData)
 #define TSDB_SUPER_TABLE_INDEX(pTable) ((pTable)->content.pIndex)
 
-SVSchema *tsdbGetTableSchema(STable *pTable);
+SSchema *tsdbGetTableSchema(STable *pTable);
 
 // ---- Operation on SMetaHandle
 #define TSDB_NUM_OF_TABLES(pHandle) ((pHandle)->numOfTables)

src/vnode/tsdb/src/tsdb.c

@@ -2,14 +2,15 @@
 #include <stdint.h>
 #include <stdlib.h>
 
-#include "taosdef.h"
+// #include "taosdef.h"
 // #include "disk.h"
+#include "tsdb.h"
 #include "tsdbCache.h"
 #include "tsdbMeta.h"
 
 typedef struct STSDBRepo {
   // TSDB configuration
-  STSDBcfg *pCfg;
+  STSDBCfg *pCfg;
 
   // The meter meta handle of this TSDB repository
   SMetaHandle *pMetaHandle;
@@ -18,12 +19,12 @@ typedef struct STSDBRepo {
   SCacheHandle *pCacheHandle;
 
   // Disk tier handle for multi-tier storage
-  SDiskTier *pDiskTier;
+  void *pDiskTier;
 
   // File Store
   void *pFileStore;
 
-  pthread_mutext_t tsdbMutex;
+  pthread_mutex_t tsdbMutex;
 
 } STSDBRepo;
 

src/vnode/tsdb/src/tsdbFileStore.c

@@ -1,6 +1,6 @@
 #include "tsdbFile.h"
 
 char *tsdbGetFileName(char *dirName, char *fname, TSDB_FILE_TYPE type){
-    char *suffix = tsdbFileSuffix[type];
+    // char *suffix = tsdbFileSuffix[type];
     // TODO 
 }

src/vnode/tsdb/src/tsdbMeta.c

@@ -1,6 +1,7 @@
 #include <stdlib.h>
 
-#include "taosdef.h"
+// #include "taosdef.h"
+#include "tsdb.h"
 #include "tsdbMeta.h"
 
 SMetaHandle *tsdbCreateMetaHandle(int32_t numOfTables) {
@@ -11,7 +12,7 @@ SMetaHandle *tsdbCreateMetaHandle(int32_t numOfTables) {
 
   pMetahandle->numOfTables = 0;
   pMetahandle->numOfSuperTables = 0;
-  pMetahandle->pTables = calloc(sizeof(STable *) * numOfTables);
+  pMetahandle->pTables = calloc(sizeof(STable *), numOfTables);
   if (pMetahandle->pTables == NULL) {
     free(pMetahandle);
     return NULL;